<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">

  <title>Data Science at the Command Line, 2e</title>

    <meta name="author" content="Jeroen Janssens" />
  
   <meta name="description" content="This thoroughly revised guide demonstrates how the flexibility of the command line can help you become a more efficient and productive data scientist. You’ll learn how to combine small yet powerful command-line tools to quickly obtain, scrub, explore, and model your data. To get you started, author Jeroen Janssens provides a Docker image packed with over 100 Unix power tools—useful whether you work with Windows, macOS, or Linux." />
   <meta name="generator" content="placeholder" />
  <meta property="og:title" content="Data Science at the Command Line, 2e" />
  <meta property="og:type" content="book" />
  <meta property="og:url" content="https://datascienceatthecommandline.com" />
  <meta property="og:image" content="https://datascienceatthecommandline.com/og.png" />
  <meta property="og:description" content="This thoroughly revised guide demonstrates how the flexibility of the command line can help you become a more efficient and productive data scientist. You’ll learn how to combine small yet powerful command-line tools to quickly obtain, scrub, explore, and model your data. To get you started, author Jeroen Janssens provides a Docker image packed with over 100 Unix power tools—useful whether you work with Windows, macOS, or Linux." />
  <meta name="twitter:card" content="summary_large_image" />
  <meta name="twitter:title" content="Data Science at the Command Line, 2e" />
  
  <meta name="twitter:description" content="This thoroughly revised guide demonstrates how the flexibility of the command line can help you become a more efficient and productive data scientist. You’ll learn how to combine small yet powerful command-line tools to quickly obtain, scrub, explore, and model your data. To get you started, author Jeroen Janssens provides a Docker image packed with over 100 Unix power tools—useful whether you work with Windows, macOS, or Linux." />
  <meta name="twitter:image" content="https://datascienceatthecommandline.com/twitter.png" />
  <!-- JS -->
  <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.6/clipboard.min.js" integrity="sha256-inc5kl9MA1hkeYUt+EC3BhlIgyp/2jDIyBLS6k3UxPI=" crossorigin="anonymous"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/fuse.js/6.4.6/fuse.js" integrity="sha512-zv6Ywkjyktsohkbp9bb45V6tEMoWhzFzXis+LrMehmJZZSys19Yxf1dopHx7WzIKxr5tK2dVcYmaCk2uqdjF4A==" crossorigin="anonymous"></script>
  <script src="https://kit.fontawesome.com/6ecbd6c532.js" crossorigin="anonymous"></script>
    <script src="libs/header-attrs-2.9/header-attrs.js"></script>
    <script src="libs/jquery-3.6.0/jquery-3.6.0.min.js"></script>
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
    <link href="libs/bootstrap-4.6.0/bootstrap.min.css" rel="stylesheet" />
    <script src="libs/bootstrap-4.6.0/bootstrap.bundle.min.js"></script>
    <link href="libs/_Source Sans Pro-0.4.0/font.css" rel="stylesheet" />
    <link href="https://fonts.googleapis.com/css2?family=Fira%20Mono:wght@400;600&amp;display=swap" rel="stylesheet"/>
    <script src="libs/bs3compat-0.3.1/transition.js"></script>
    <script src="libs/bs3compat-0.3.1/tabs.js"></script>
    <script src="libs/bs3compat-0.3.1/bs3compat.js"></script>
    <link href="libs/bs4_book-1.0.0/bs4_book.css" rel="stylesheet" />
    <script src="libs/bs4_book-1.0.0/bs4_book.js"></script>
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
    <link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
    <link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
    <link rel="manifest" href="/site.webmanifest">
    <link rel="mask-icon" href="/safari-pinned-tab.svg" color="#d42d2d">
    <meta name="apple-mobile-web-app-title" content="Data Science at the Command Line">
    <meta name="application-name" content="Data Science at the Command Line">
    <meta name="msapplication-TileColor" content="#b91d47">
    <meta name="theme-color" content="#ffffff">
    <script>
      (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
      })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
      ga('create', 'UA-43246574-3', 'auto');
      ga('send', 'pageview');
    </script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/autocomplete.js/0.38.0/autocomplete.jquery.min.js" integrity="sha512-GU9ayf+66Xx2TmpxqJpliWbT5PiGYxpaG8rfnBEk1LL8l1KGkRShhngwdXK1UgqhAzWpZHSiYPc09/NwDQIGyg==" crossorigin="anonymous"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/mark.js/8.11.1/mark.min.js" integrity="sha512-5CYOlHXGh6QpOFA/TeTylKLWfB3ftPsde7AnmhuitiTX4K5SqCLBeKro6sPS8ilsz1Q4NRx3v8Ko2IBiszzdww==" crossorigin="anonymous"></script>

  <!-- CSS -->
    <link rel="stylesheet" href="dsatcl2e.css" />
  
</head>

<body data-spy="scroll" data-target="#toc">

<div class="container-fluid">
<!--bookdown:title:start-->
<!--bookdown:title:end-->

<!--bookdown:toc:start-->
<div class="row">
  <header class="col-sm-12 col-lg-2 sidebar sidebar-book">
    <a class="sr-only sr-only-focusable" href="#content">Skip to main content</a>

    <div class="d-flex align-items-start justify-content-between">
      <img id="cover" class="d-none d-lg-block" src="images/cover-small.png">
      <h1 class="d-lg-none">
        <a href="index.html" title="">Data Science at the Command Line, 2e</a>
      </h1>
      <button class="btn btn-outline-primary d-lg-none ml-2 mt-1" type="button" data-toggle="collapse" data-target="#main-nav" aria-expanded="true" aria-controls="main-nav"><i class="fas fa-bars"></i><span class="sr-only">Show table of contents</span></button>
    </div>

    <div id="main-nav" class="collapse-lg">
      <form role="search">
        <input id="search" class="form-control" type="search" placeholder="Search" aria-label="Search">
      </form>
      <nav aria-label="Table of contents">
        <h2>Table of contents</h2>
        <div id="book-toc"></div>

        <div class="book-extra">
          <p><a id="book-repo" href="#">View book repository <i class="fab fa-github"></i></a></li></p>
        </div>

        <div>
          <a id="course-signup" href="/#course">Embrace the Command Line</a>
        </div>
      </nav>
    </div>
  </header>

  <main class="col-sm-12 col-md-9 col-lg-7" id="content">
<!--bookdown:toc:end-->
<!--bookdown:body:start-->
<div id="welcome" class="section level1 unnumbered">
<h1 class="unnumbered">Welcome</h1>
<div class="h1" style="margin-top: 1.5rem;">
Data Science at the Command Line
</div>
<div class="h4">
Obtain, Scrub, Explore, and Model Data with Unix Power Tools
</div>
<div class="cover-in-text">
<p><img class="d-block d-lg-none" src="images/cover-small.png"></p>
</div>
<p>Welcome to the website of the second edition of <em>Data Science at the Command Line</em> by Jeroen Janssens, published by O’Reilly Media in October 2021. This website is free to use. The contents is licensed under the <a href="https://creativecommons.org/licenses/by-nc-nd/4.0/">Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License</a>. You can order a physical copy at <a href="https://www.amazon.com/Data-Science-Command-Line-Explore-dp-1492087912/dp/1492087912">Amazon</a>.</p>
<p>Want to learn from Jeroen in person? Through his company, Data Science Workshops, Jeroen provides in-company training about data science at the command line and related topics such as Python, R, and machine learning. Visit <a href="https://datascienceworkshops.com">Data Science Workshops</a> for more information.</p>

<div class="rmdtip">
Jeroen is currently working on a new course <a href="/#course">Embrace the Command Line</a>. If you haven’t fully embraced the command line yet, then this course might be for you. The beta cohort is expected to start in Q1 2021. You can learn more about this course and tell Jeroen what you think <a href="/#course">here</a>.
</div>
<div id="description" class="section level2 unnumbered">
<h2 class="unnumbered">Description</h2>
<p>This thoroughly revised guide demonstrates how the flexibility of the command line can help you become a more efficient and productive data scientist. You’ll learn how to combine small yet powerful command-line tools to quickly obtain, scrub, explore, and model your data. To get you started, author Jeroen Janssens provides a Docker image packed with over 100 Unix power tools—useful whether you work with Windows, macOS, or Linux.</p>
<p>You’ll quickly discover why the command line is an agile, scalable, and extensible technology. Even if you’re comfortable processing data with Python or R, you’ll learn how to greatly improve your data science workflow by leveraging the command line’s power. This book is ideal for data scientists, analysts, engineers, system administrators, and researchers.</p>
<ul>
<li>Obtain data from websites, APIs, databases, and spreadsheets</li>
<li>Perform scrub operations on text, CSV, HTML, XML, and JSON files</li>
<li>Explore data, compute descriptive statistics, and create visualizations</li>
<li>Manage your data science workflow</li>
<li>Create your own tools from one-liners and existing Python or R code</li>
<li>Parallelize and distribute data-intensive pipelines</li>
<li>Model data with dimensionality reduction, regression, and classification algorithms</li>
<li>Leverage the command line from Python, Jupyter, R, RStudio, and Apache Spark</li>
</ul>

<div class="rmdnote">
If you find this book helpful, consider spreading the word! You could, for instance,
share it on <a href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fdatascienceatthecommandline.com&amp;via=jeroenhjanssens&amp;text=Data%20Science%20at%20the%20Command%20Line%2C%20second%20edition">Twitter</a>,
write a review on <a href="https://www.amazon.com/Data-Science-Command-Line-Explore-dp-1492087912/dp/1492087912">Amazon</a>, or
star the <a href="https://github.com/jeroenjanssens/data-science-at-the-command-line">Github repository</a>. Much appreciated!
</div>
</div>
<div id="praise" class="section level2 unnumbered">
<h2 class="unnumbered">Praise</h2>
<blockquote>
<p>
Traditional computer and data science curricula all too often mistake the command line as an obsolete relic instead of teaching it as the modern and vital toolset that it is. Only well into my career did I come to grasp the elegance and power of the command line <span class="keep-together">for easily</span> exploring messy datasets and even creating reproducible data pipelines <span class="keep-together">for work. The</span> first edition of <em>Data Science at the Command Line</em> was one of the <span class="keep-together">most comprehensive and clear</span> references when I was a novice in the art, and now <span class="keep-together">with the second edition,</span> I’m again learning new tools and applications from it.
</p>
<p data-type="attribution">
<strong>Dan Nguyen</strong>, data scientist, former news application developer at ProPublica, and former Lorry I. Lokey Visiting Professor in <span class="keep-together">Professional Journalism at Stanford University</span>
</p>
</blockquote>
<blockquote>
<p>
The Unix philosophy of simple tools, each doing one job well, then cleverly piped <span class="keep-together">together, is</span> embodied by the command line. Jeroen expertly discusses how to <span class="keep-together">bring that philosophy</span> into your work in data science, illustrating how the <span class="keep-together">command line is not only the</span> world of file input/output, but also the <span class="keep-together">world of data manipulation, exploration, and even modeling.</span>
</p>
<p data-type="attribution">
<strong>Chris H. Wiggins</strong>, associate professor in the department of applied physics and applied mathematics at Columbia University, <span class="keep-together">and chief data scientist at <span class="plain">The New York Times</span></span>
</p>
</blockquote>
<blockquote>
<p>
This book explains how to integrate common data science tasks into a <span class="keep-together">coherent workflow. It’s</span> not just about tactics for breaking down problems, <span class="keep-together">it’s also about strategies for assembling the pieces of the solution.</span>
</p>
<p data-type="attribution">
<strong>John D. Cook</strong>, consultant in applied mathematics, <span class="keep-together">statistics, and technical computing</span>
</p>
</blockquote>
<blockquote class="pagebreak-before">
<p>
Despite what you may hear, most practical data science is still focused on interesting <span class="keep-together">visualizations and insights</span> derived from flat files. Jeroen’s book leans into this <span class="keep-together">reality, and helps</span> reduce complexity for data practitioners by showing how <span class="keep-together">time-tested command-line tools</span> can be repurposed for data science.
</p>
<p data-type="attribution">
<strong>Paige Bailey</strong>, principal product manager <span class="keep-together">code intelligence at Microsoft, GitHub</span>
</p>
</blockquote>
<blockquote>
<p>
It’s amazing how fast so much data work can be performed at the command line <span class="keep-together">before ever pulling</span> the data into R, Python, or a database. Older technologies like <span class="keep-together">sed and awk are still</span> incredibly powerful and versatile. Until I read <em>Data Science <span class="keep-together">at the Command Line</span></em>, I had only heard of these tools but never saw their full power. <span class="keep-together">Thanks to Jeroen,</span> it’s like I now have a secret weapon for working with large data.
</p>
<p data-type="attribution">
<strong>Jared Lander</strong>, chief data scientist at Lander Analytics, organizer of the New York Open Statistical Programming Meetup, <span class="keep-together">and author of <span class="plain">R for Everyone</span></span>
</p>
</blockquote>
<blockquote>
<p>
The command line is an essential tool in every data scientist’s toolbox, <span class="keep-together">and knowing it well</span> makes it easy to translate questions you have of your <span class="keep-together">data to real-time insights. Jeroen</span> not only explains the basic Unix philosophy <span class="keep-together">of how to chain together single-purpose</span> tools to arrive at simple solutions <span class="keep-together">for complex problems, but also</span> introduces new command-line tools <span class="keep-together">for data cleaning, analysis, visualization, and modeling</span>.
</p>
<p data-type="attribution">
<strong>Jake Hofman</strong>, senior principal researcher at <span class="keep-together">Microsoft Research,</span> and adjunct assistant professor in the <span class="keep-together">department of applied mathematics at Columbia University</span>
</p>
</blockquote>
</div>
<div id="dedication" class="section level2 unnumbered">
<h2 class="unnumbered">Dedication</h2>
<div style="text-align: center;">
<p><em>Once again to my wife, Esther. Without her continued encouragement, support,<br/>
and patience, this second edition would surely have ended up in</em> /dev/null<em>.</em></p>
</div>
</div>
<div id="about-the-author" class="section level2 unnumbered">
<h2 class="unnumbered">About the Author</h2>
<p><strong>Jeroen Janssens</strong> is an independent data science consultant and instructor. He enjoys visualizing data, implementing machine learning models, and building solutions using Python, R, JavaScript, and Bash. Jeroen manages <a href="https://datascienceworkshops.com">Data Science Workshops</a>, a training and coaching firm that organizes open enrollment workshops, in-company courses, inspiration sessions, hackathons, and meetups. Previously, he was an
assistant professor at Jheronimus Academy of Data Science and a data scientist at Elsevier in Amsterdam and various startups in New York City. Jeroen holds a PhD in machine learning from Tilburg University and an MSc in artificial intelligence from Maastricht University. He lives with his wife and two kids in Rotterdam, the Netherlands.
You can find Jeroen on <a href="https://twitter.com/jeroenhjanssens">Twitter</a>, <a href="https://github.com/jeroenjanssens">GitHub</a>, and <a href="https://www.linkedin.com/in/jeroenjanssens/">LinkedIn</a>.</p>
</div>
<div id="colophon" class="section level2 unnumbered">
<h2 class="unnumbered">Colophon</h2>
<p>The animal on the cover of <em>Data Science at the Command Line</em> is a wreathed hornbill (<em>Rhytidoceros undulatus</em>). Also known as the bar-pouched wreathed hornbill, the species is found in forests in mainland Southeast Asia and in northeastern India and Bhutan. Hornbills are named for the <em>casques</em> that form on the upper part of the birds’ bills. No single obvious purpose exists for these hollow, keratizined structures, but they may serve as a means of recognition between members of the species, as an amplifier for the birds’ calls, or—because males often exhibit larger casques than females of the species—for gender recognition. Wreathed hornbills can be distinguished from plain-pouched hornbills, to whom they are closely related and otherwise similar in appearance, by a dark bar on the lower part of the wreathed hornbills’ throats.</p>
<p>Wreathed hornbills roost in flocks of up to four hundred but mate in monogamous, lifelong partnerships. With help from the males, females seal themselves up in tree cavities behind dung and mud to lay eggs and brood. Through a slit large enough for his beak alone, the male feeds his mate and their young for up to four months. A diet of animal prey becomes predominantly fruit when females and their young leave the nest. Hornbill couples have been known to return to the same nest for as many as nine years.</p>
<p>Many of the animals on O’Reilly covers are endangered; all of them are important to the world.</p>
<p>The color illustration is by Karen Montgomery, based on a black and white engraving from Braukhaus’s <em>Lexicon</em>. The cover fonts are Gilroy Semibold and Guardian Sans. The text and heading font is Source Sans Pro and the code font is Fira Mono.</p>
<!--chapter:end:index.Rmd-->
<!--A[foreword]
A-->
</div>
</div>
<div id="foreword" class="section level1 unnumbered">
<h1 class="unnumbered">Foreword</h1>
<p>It was love at first sight.</p>
<p>It must have been around 1981 or 1982 that I got my first taste of Unix.
Its command-line shell, which uses the same language for single commands and complex programs, changed my world, and I never looked back.</p>
<p>I was a writer who had discovered the joys of computing, and regular expressions were my gateway drug.
I’d first tried them in the text editor in HP’s RTE operating system, but it was only when I came to Unix and its philosophy of small cooperating tools with the command line shell as the glue that tied them together that I fully understood their power.
Regular expressions in <code>ed</code>, <code>ex</code>, <code>vi</code> (now <code>vim</code>), and <code>emacs</code> were powerful, sure, but it wasn’t until I saw how <code>ex</code> scripts unbound became <code>sed</code>, the Unix stream editor, and then AWK, which allowed you to bind programmed actions to regular expressions, and how shell scripts let you build pipelines not only out of the existing tools but out of new ones you’d written yourself, that I really got it.
Programming is how you speak with computers, how you tell them what you want them to do, not just once, but in ways that persist, in ways that can be varied like human language, with repeatable structure but different verbs and objects.</p>
<p>As a beginner, other forms of programming seemed more like recipes to be followed exactly, careful incantations where you had to get everything right, or like waiting for a teacher to grade an essay you’d written.
With shell programming, there was no compilation and waiting.
It was more like a conversation with a friend.
When the friend didn’t understand, you could easily try again.
What’s more, if you had something simple to say, you could just say it with one word.
And there were already words for a whole lot of the things you might want to say.
But if there weren’t, you could easily make up new words.
And you could string the words you learned and the words you made up into gradually more complex sentences, paragraphs, and eventually get to persuasive essays.</p>
<p>Almost every other programming language is more powerful than the shell and its associated tools, but for me at least, none provides an easier pathway into the programming mindset, and none provides a better environment for a kind of everyday conversation with the machines that we ask to help us with our work.
As Brian Kernighan, one of the creators of AWK as well as the co-author of the marvelous book <em>The Unix Programming Environment</em>, said in his 2019 interview with Lex Fridman, “[Unix] was meant to be an environment where it was really easy to write programs.” <a href="https://www.happyscribe.com/public/lex-fridman-podcast-artificial-intelligence-ai/109-brian-kernighan-unix-c-awk-ampl-and-go-programming#paragraph_1371">[00:23:10]</a>
He went on to explain why he often still uses AWK rather than writing a Python program when he’s exploring data.
“It doesn’t scale to big programs, but it does pretty darn well on these little things where you just want to see all the somethings in something.” <a href="https://www.happyscribe.com/public/lex-fridman-podcast-artificial-intelligence-ai/109-brian-kernighan-unix-c-awk-ampl-and-go-programming#paragraph_2221">[00:37:01]</a></p>
<p>In <em>Data Science at the Command Line</em>, Jeroen Janssens demonstrates just how powerful the Unix/Linux approach to the command line is even today.
If Jeroen hadn’t already done so, I’d write an essay here about just why the command line is such a sweet and powerful match with the kinds of tasks so often encountered in data science.
But he already starts out his book by explaining that.
So I’ll just say this: the more you use the command line, the more often you will find yourself coming back to it as the easiest way to do much of your work.
And whether you’re a shell newbie, or just someone who hasn’t thought much about what a great fit shell programming is for data science, this is a book you will come to treasure.
Jeroen is a great teacher, and the material he covers is priceless.</p>
<p style="text-align: right; font-style: italic">
—Tim O’Reilly<br />
May 2021
</p>
<!--chapter:end:foreword.Rmd-->
<!--A[preface]
A-->
</div>
<div id="preface" class="section level1 unnumbered">
<h1 class="unnumbered">Preface</h1>
<p>Data science is an exciting field to work in.
It’s also still relatively young.
Unfortunately, many people, and many companies as well, believe that you need new technology to tackle the problems posed by data science.
However, as this book demonstrates, many things can be accomplished by using the command line instead, and sometimes in a much more efficient way.</p>
<p>During my PhD program, I gradually switched from using Microsoft Windows to using Linux.
Because this transition was a bit scary at first, I started with having both operating systems installed next to each other (known as a dual-boot).
The urge to switch back and forth between Microsoft Windows and Linux eventually faded, and at some point I was even tinkering around with Arch Linux, which allows you to build up your own custom Linux machine from scratch.
All you’re given is the command line, and it’s up to you what to make of it.
Out of necessity, I quickly became very comfortable using the command line.
Eventually, as spare time got more precious, I settled down with a Linux distribution known as Ubuntu because of its ease of use and large community.
However, the command line is still where I’m spending most of my time.</p>
<p>It actually wasn’t too long ago that I realized that the command line is not just for installing software, configuring systems, and searching files.
I started learning about tools such as <code>cut</code>, <code>sort</code>, and <code>sed</code>.
These are examples of command-line tools that take data as input, do something to it, and print the result.
Ubuntu comes with quite a few of them.
Once I understood the potential of combining these small tools, I was hooked.</p>
<p>After earning my PhD, when I became a data scientist, I wanted to use this approach to do data science as much as possible.
Thanks to a couple of new, open source command-line tools including <code>xml2json</code>, <code>jq</code>, and <code>json2csv</code>, I was even able to use the command line for tasks such as scraping websites and processing lots of JSON data.</p>
<p>In September 2013, I decided to write a blog post titled <a href="http://www.jeroenjanssens.com/2013/09/19/seven-command-line-tools-for-data-science.html">Seven Command-line Tools for Data Science</a>.
To my surprise, the blog post got quite some attention, and I received a lot of suggestions of other command-line tools.
I started wondering whether the blog post could be turned into a book.
I was pleased that, some 10 months later, and with the help of many talented people (see the acknowledgments), the answer was yes.</p>
<p>I am sharing this personal story not so much because I think you should know how this book came about, but because I want to you know that I had to learn about the command line as well.
Because the command line is so different from using a graphical user interface, it can seem scary at first.
But if I could learn it, then you can as well.
No matter what your current operating system is and no matter how you currently work with data, after reading this book you will be able to do data science at the command line.
If you’re already familiar with the command line, or even if you’re already dreaming in shell scripts, chances are that you’ll still discover a few interesting tricks or command-line tools to use for your next data science project.</p>
<div id="what-to-expect-from-this-book" class="section level2 unnumbered">
<h2 class="unnumbered">What to Expect from This Book</h2>
<p>In this book, we’re going to obtain, scrub, explore, and model data—a lot of it.
This book is not so much about how to become <em>better</em> at those data science tasks.
There are already great resources available that discuss, for example, when to apply which statistical test or how data can best be visualized.
Instead, this practical book aims to make you more <em>efficient</em> and <em>productive</em> by teaching you how to perform those data science tasks at the command line.</p>
<p>While this book discusses more than 90 command-line tools, it’s not the tools themselves that matter most.
Some command-line tools have been around for a very long time, while others will be replaced by better ones.
New command-line tools are being created even as you’re reading this.
Over the years, I have discovered many amazing command-line tools.
Unfortunately, some of them were discovered too late to be included in the book.
In short, command-line tools come and go.
But that’s OK.</p>
<p>What matters most is the underlying idea of working with tools, pipes, and data.
Most command-line tools do one thing and do it well.
This is part of the Unix philosophy, which makes several appearances throughout the book.
Once you have become familiar with the command line, know how to combine command-line tools, and can even create new ones, you have developed an invaluable skill.</p>
</div>
<div id="changes-for-the-second-edition" class="section level2 unnumbered">
<h2 class="unnumbered">Changes for the Second Edition</h2>
<p>While the command line as a technology and as a way of working is timeless, some of the tools discussed in the first edition have either been superseded by newer tools (e.g., <code>csvkit</code> has largely been replaced by <code>xsv</code>) or abandoned by their developers (e.g., <code>drake</code>), or they’ve been suboptimal choices (e.g., <code>weka</code>).
I have learned a lot since the first edition was published in October 2014, either through my own experience or as a result of the useful feedback from my readers.
Even though the book is quite niche because it lies at the intersection of two subjects, there remains a steady interest from the data science community, as evidenced by the many positive messages I receive almost every day.
By updating the first edition, I hope to keep the book relevant for at least another five years.
Here’s a nonexhaustive list of changes I have made:</p>
<ul>
<li>I replaced <code>csvkit</code> with <code>xsv</code> as much as possible. <code>xsv</code> is a much faster alternative to working with CSV files.</li>
<li>In Section 2.2 and 3.2 I replaced the VirtualBox image with a Docker image. Docker is a faster and more lightweight way of running an isolated environment than VirtualBox.</li>
<li>I now use <code>pup</code> instead of <code>scrape</code> to work with HTML. <code>scrape</code> is a Python tool I created myself. <code>pup</code> is much faster, has more features, and is easier to install.</li>
<li><a href="#chapter-6-project-management-with-make">Chapter 6</a> has been rewritten from scratch. Instead of <code>drake</code> I now use <code>make</code> to do project management. <code>drake</code> is no longer maintained and <code>make</code> is much more mature and very popular with developers.</li>
<li>I replaced <code>Rio</code> with <code>rush</code>. <code>Rio</code> is a clunky Bash script I created myself. <code>rush</code> is an R package that is a much more stable and flexible way of using R from the command line.</li>
<li>In <a href="#chapter-9-modeling-data">Chapter 9</a> I replaced Weka and BigML with Vowpal Wabbit (<code>vw</code>). Weka is old and the way it is used from the command line is clunky. BigML is a commercial API on which I no longer want to rely. Vowpal Wabbit is a very mature machine learning tool, developed at Yahoo! and now at Microsoft.</li>
<li><a href="#chapter-10-polyglot-data-science">Chapter 10</a> is an entirely new chapter about integrating the command line into existing workflows, including Python, R, and Apache Spark. In the first edition I mentioned that the command line can easily be integrated with existing workflows, but I never got into that. This chapter fixes that.</li>
</ul>
</div>
<div id="how-to-read-this-book" class="section level2 unnumbered">
<h2 class="unnumbered">How to Read This Book</h2>
<p>In general, I advise you to read this book in a linear fashion.
Once a concept or command-line tool has been introduced, chances are that I employ it in a later chapter.
For example, in <a href="#chapter-9-modeling-data">Chapter 9</a>, I make heavy use of <code>parallel</code>, which is introduced extensively in <a href="#chapter-8-parallel-pipelines">Chapter 8</a>.</p>
<p>Data science is a broad field that intersects many other fields such as programming, data visualization, and machine learning.
As a result, this book touches on many interesting topics which unfortunately cannot be discussed at full length.
Throughout the book, at the end of each chapter, there are suggestions for further exploration.
It’s not required to read this material in order to follow along with the book, but when you are interested, you know that there’s much more to learn.</p>
</div>
<div id="who-this-book-is-for" class="section level2 unnumbered">
<h2 class="unnumbered">Who This Book Is For</h2>
<p>This book makes just one assumption about you: that you work with data.
It doesn’t matter which programming language or statistical computing environment you’re currently using.
The book explains all the necessary concepts from the beginning.</p>
<p>It also doesn’t matter whether your operating system is Microsoft Windows, macOS, or some flavor of Linux.
The book comes with a Docker image, which is an easy-to-install virtual environment.
It allows you to run the command-line tools and follow along with the code examples in the same environment as this book was written.
You don’t have to waste time figuring out how to install all the command-line tools and their dependencies.</p>
<p>The book contains some code in Bash, Python, and R, so it’s helpful if you have some programming experience, but it’s by no means required to follow along with the examples.</p>
</div>
<div id="conventions-used-in-this-book" class="section level2 unnumbered">
<h2 class="unnumbered">Conventions Used in This Book</h2>
<p>The following typographical conventions are used in this book:</p>
<dl>
<dt><em>Italic</em></dt>
<dd><p>Indicates new terms, URLs, directory names, and filenames.</p>
</dd>
<dt><code>Constant width</code></dt>
<dd><p>Used for code and commands, as well as within paragraphs to refer to command-line tools and their options.</p>
</dd>
<dt><strong><code>Constant width bold</code></strong></dt>
<dd><p>Shows commands or other text that should be typed literally by the user.</p>
</dd>
</dl>

<div class="rmdtip">
This element signifies a tip or suggestion.
</div>

<div class="rmdnote">
This element signifies a general note.
</div>

<div class="rmdcaution">
This element indicates a warning or caution.
</div>
<!--A
=== Using Code Examples

Supplemental material (code examples, exercises, etc.) is available for download at link:$$https://github.com/jeroenjanssens/data-science-at-the-command-line$$[].

This book is here to help you get your job done. In general, if example code is offered with this book, you may use it in your programs and documentation. You do not need to contact us for permission unless you’re reproducing a significant portion of the code. For example, writing a program that uses several chunks of code from this book does not require permission. Selling or distributing examples from O’Reilly books does require permission. Answering a question by citing this book and quoting example code does not require permission. Incorporating a significant amount of example code from this book into your product’s documentation does require permission.

We appreciate, but generally do not require, attribution. An attribution usually includes the title, author, publisher, and ISBN. For example: “Data Science at the Command Line by Jeroen Janssens (O’Reilly). Copyright 2021 Jeroen Janssens, 978-1-492-08791-5.”

If you feel your use of code examples falls outside fair use or the permission given above, feel free to contact us at pass:[<a class="email" href="mailto:permissions@oreilly.com"><em>permissions@oreilly.com</em></a>].

=== O'Reilly Online Learning

[role = "ormenabled"]
[NOTE]
====
For more than 40 years, pass:[<a href="http://oreilly.com" class="orm:hideurl"><em class="hyperlink">O’Reilly Media</em></a>] has provided technology and business training, knowledge, and insight to help companies succeed.
====

Our unique network of experts and innovators share their knowledge and expertise through books, articles, and our online learning platform. O’Reilly’s online learning platform gives you on-demand access to live training courses, in-depth learning paths, interactive coding environments, and a vast collection of text and video from O'Reilly and 200+ other publishers. For more information, visit pass:[<a href="http://oreilly.com" class="orm:hideurl"><em>http://oreilly.com</em></a>].

=== How to Contact Us

Please address comments and questions concerning this book to the publisher:

++++
<ul class="simplelist">
  <li>O’Reilly Media, Inc.</li>
  <li>1005 Gravenstein Highway North</li>
  <li>Sebastopol, CA 95472</li>
  <li>800-998-9938 (in the United States or Canada)</li>
  <li>707-829-0515 (international or local)</li>
  <li>707-829-0104 (fax)</li>
</ul>
++++

We have a web page for this book, where we list errata, examples, and any additional information. You can access this page at link:$$https://wwww.datascienceatthecommandline.com$$[].

Email pass:[<a class="email" href="mailto:bookquestions@oreilly.com"><em>bookquestions@oreilly.com</em></a>] to comment or ask technical questions about this book.

For news and information about our books and courses, visit link:$$http://oreilly.com$$[].

Find us on Facebook: link:$$http://facebook.com/oreilly$$[]

Follow us on Twitter: link:$$http://twitter.com/oreillymedia$$[]

Watch us on YouTube: link:$$http://www.youtube.com/oreillymedia$$[]



A-->
</div>
<div id="acknowledgments" class="section level2 unnumbered">
<h2 class="unnumbered">Acknowledgments</h2>
<div id="acknowledgments-for-the-second-edition-2021" class="section level3 unnumbered">
<h3 class="unnumbered">Acknowledgments for the Second Edition (2021)</h3>
<p>Seven years have passed since the first edition came out.
During this time, and especially during the last 13 months, many people have helped me.
Without them, I would have never been able to write a second edition.</p>
<p>I was once again blessed with three wonderful editors at O’Reilly.
I would like to thank Sarah “Embrace the deadline” Grey, Jess “Pedal to the metal” Haberman, and Kate “Let it go” Galloway. Their middle names say it all. With their incredible help, I was able to embrace the deadlines, put the pedal to metal when it mattered, and eventually let it go.
I’d also like to thank their colleagues Angela Rufino, Arthur Johnson, Cassandra Furtado, David Futato, Helen Monroe, Karen Montgomery, Kate Dullea, Kristen Brown, Marie Beaugureau, Marsee Henon, Nick Adams, Regina Wilkinson, Shannon Cutt, Shannon Turlington, and Yasmina Greco, for making the collaboration with O’Reilly such a pleasure.</p>
<p>Despite having an automated process to execute the code and paste back the results (thanks to R Markdown and Docker), the number of mistakes I was able to make is impressive.
Thank you Aaditya Maruthi, Brian Eoff, Caitlin Hudon, Julia Silge, Mike Dewar, and Shane Reustle for reducing this number immensely.
Of course, any mistakes left are my responsibility.</p>
<p>Marc Canaleta deserves a special thank you.
In October 2014, shortly after the first edition came out, Marc invited me to give a one-day workshop about <em>Data Science at the Command Line</em> to his team at Social Point in Barcelona.
Little did we both know that many workshops would follow.
It eventually led me to start my own company: Data Science Workshops.
Every time I teach, I learn something new.
They probably don’t know it, but each student has had an impact, in one way or another, on this book.
To them I say: thank you.
I hope I can teach for a very long time.</p>
<p>Captivating conversations, splendid suggestions, and passionate pull requests.
I greatly appreciate each and every contribution by following generous people:
Adam Johnson,
Andre Manook,
Andrea Borruso,
Andres Lowrie,
Andrew Berisha,
Andrew Gallant,
Andrew Sanchez,
Anicet Ebou,
Anthony Egerton,
Ben Isenhart,
[.keep-together]#Chris Wiggins#,
Chrys Wu,
Dan Nguyen,
Darryl Amatsetam,
Dmitriy Rozhkov,
Doug Needham,
Edgar Manukyan,
Erik Swan,
Felienne Hermans,
George Kampolis,
Giel van Lankveld,
Greg Wilson,
Hay Kranen,
Ioannis Cherouvim,
Jake Hofman,
Jannes Muenchow,
Jared Lander,
Jay Roaf,
Jeffrey Perkel,
Jim Hester,
Joachim Hagege,
Joel Grus,
John Cook,
John Sandall,
Joost Helberg,
Joost van Dijk,
Joyce Robbins,
Julian Hatwell,
Karlo Guidoni,
Karthik Ram,
Lissa Hyacinth,
Longhow Lam,
Lui Pillmann,
Lukas Schmid,
Luke Reding,
Maarten van Gompel,
Martin Braun,
Max Schelker,
Max Shron,
Nathan Furnal,
Noah Chase,
Oscar Chic,
Paige Bailey,
Peter Saalbrink,
Rich Pauloo,
Richard Groot,
Rico Huijbers,
Rob Doherty,
Robbert van Vlijmen,
Russell Scudder,
Sylvain Lapoix,
TJ Lavelle,
Tan Long,
Thomas Stone,
Tim O’Reilly,
Vincent Warmerdam, and
Yihui Xie.</p>
<p>Throughout this book, and especially in the footnotes and appendix, you’ll find hundreds of names.
These names belong to the authors of the many tools, books, and other resources on which this book stands.
I’m incredibly grateful for their hard work, regardless of whether that work was done 50 years or 50 days ago.</p>
<p>Above all, I would like to thank my wife Esther, my daughter Florien, and my son Olivier for reminding me daily what truly matters.
I promise it’ll be a few years before I start writing the third edition.</p>
</div>
<div id="acknowledgments-for-the-first-edition-2014" class="section level3 unnumbered">
<h3 class="unnumbered">Acknowledgments for the First Edition (2014)</h3>
<p>First of all, I’d like to thank Mike Dewar and Mike Loukides for believing that my blog post <a href="http://jeroenjanssens.com/2013/09/19/seven-command-line-tools-for-data-science.html">Seven Command-Line Tools for Data Science</a>, which I wrote in September 2013, could be expanded into a book.</p>
<p>Special thanks to my technical reviewers Mike Dewar, Brian Eoff, and Shane Reustle for reading various drafts, meticulously testing all the commands, and providing invaluable feedback.
Your efforts have improved the book greatly.
The remaining errors are entirely my own responsibility.</p>
<p>I had the privilege of working together with three amazing editors, namely: Ann Spencer, Julie Steele, and Marie Beaugureau.
Thank you for your guidance and for being such great liaisons with the many talented people at O’Reilly.
Those people include: Laura Baldwin, Huguette Barriere, Sophia DeMartini, Yasmina Greco, Rachel James, Ben Lorica, Mike Loukides, and Christopher Pappas.
There are many others whom I haven’t met because they are operating behind the scenes.
Together they ensured that working with O’Reilly has truly been a pleasure.</p>
<p>This book discusses over 80 command-line tools.
Needless to say, without these tools, this book wouldn’t have existed in the first place.
I’m therefore extremely grateful to all the authors who created and contributed to these tools.
The complete list of authors is unfortunately too long to include here; they are mentioned in the Appendix.
Thanks especially to Aaron Crow, Jehiah Czebotar, Christoph Groskopf, Dima Kogan, Sergey Lisitsyn, Francisco J.
Martin, and Ole Tange for providing help with their amazing command-line tools.</p>
<p>Eric Postma and Jaap van den Herik, who supervised me during my PhD program, deserve a special thank you.
Over the course of five years they have taught me many lessons.
Although writing a technical book is quite different from writing a PhD thesis, many of those lessons proved to be very helpful in the past nine months as well.</p>
<p>Finally, I’d like to thank my colleagues at YPlan, my friends, my family, and especially my wife Esther for supporting me and for pulling me away from the command line at just the right times.</p>
<!--chapter:end:preface.Rmd-->
<!--A[role="pagenumrestart"]
A-->
</div>
</div>
</div>
<div id="chapter-1-introduction" class="section level1" number="1">
<h1 number="1"><span class="header-section-number">1</span> Introduction</h1>
<p>This book is about doing data science at the command line.
My aim is to make you a more efficient and productive data scientist by teaching you how to leverage the power of the command line.</p>
<p>Having both the terms <em>data science</em> and <em>command line</em> in the title requires an explanation.
How can a technology that is over 50 years old<a href="#fn1" class="footnote-ref" id="fnref1"><sup>1</sup></a> be of any use to a field that is only a few years young?</p>
<p>Today, data scientists can choose from an overwhelming collection of exciting technologies and programming languages.
Python, R, Julia, and Apache Spark are but a few examples.
You may already have experience in one or more of these.
If so, then why should you still care about the command line for doing data science?
What does the command line have to offer that these other technologies and programming languages do not?</p>
<p>These are all valid questions.
In this first chapter I will answer these questions as follows.
First, I provide a practical definition of data science that will act as the backbone of this book.
Second, I’ll list five important advantages of the command line.
By the end of this chapter I hope to have convinced you that the command line is indeed worth learning for doing data science.</p>
<div id="data-science-is-osemn" class="section level2" number="1.1">
<h2 number="1.1"><span class="header-section-number">1.1</span> Data Science is OSEMN</h2>
<p>The field of data science is still in its infancy, and as such, there exist various definitions of what it encompasses.
Throughout this book I employ a very practical definition by <span class="citation"><a href="#ref-Mason2010" role="doc-biblioref">Hilary Mason and Chris H. Wiggins</a><a href="#fn2" class="footnote-ref" id="fnref2"><sup>2</sup></a></span>.
They define data science according to the following five steps: (1) obtaining data, (2) scrubbing data, (3) exploring data, (4) modeling data, and (5) interpreting data.
Together, these steps form the OSEMN model (which is pronounced as <em>awesome</em>).
This definition serves as the backbone of this book because each step, (except step 5, interpreting data, which I explain below) has its own chapter.</p>
<p>Although the five steps are discussed in a linear and incremental fashion, in practice it is very common to move back and forth between them or to perform multiple steps at the same time.
Figure @ref(fig:diagram-osemn) illustrates that doing data science is an iterative and non-linear process.
For example, once you have modeled your data, and you look at the results, you may decide to go back to the scrubbing step to the adjust the features of the dataset.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0101.png" alt="Doing data science is an iterative and non-linear process" width="90%" />
<p class="caption">
(#fig:diagram-osemn)Doing data science is an iterative and non-linear process
</p>
</div>
<p>Below I explain what each step entails.</p>
<div id="obtaining-data" class="section level3" number="1.1.1">
<h3 number="1.1.1"><span class="header-section-number">1.1.1</span> Obtaining Data</h3>
<p>Without any data, there is little data science you can do.
So the first step is obtaining data.
Unless you are fortunate enough to already possess data, you may need to do one or more of the following:</p>
<ul>
<li>Download data from another location (e.g., a webpage or server)</li>
<li>Query data from a database or API (e.g., MySQL or Twitter)</li>
<li>Extract data from another file (e.g., an HTML file or spreadsheet)</li>
<li>Generate data yourself (e.g., reading sensors or taking surveys)</li>
</ul>
<p>In <a href="#chapter-3-obtaining-data">Chapter 3</a>, I discuss several methods for obtaining data using the command line.
The obtained data will most likely be in either plain text, CSV, JSON, HTML, or XML format.
The next step is to scrub this data.</p>
</div>
<div id="scrubbing-data" class="section level3" number="1.1.2">
<h3 number="1.1.2"><span class="header-section-number">1.1.2</span> Scrubbing Data</h3>
<p>It is not uncommon that the obtained data has missing values, inconsistencies, errors, weird characters, or uninteresting columns.
In that case, you have to <em>scrub</em>, or clean, the data before you can do anything interesting with it.
Common scrubbing operations include:</p>
<ul>
<li>Filtering lines</li>
<li>Extracting certain columns</li>
<li>Replacing values</li>
<li>Extracting words</li>
<li>Handling missing values and duplicates</li>
<li>Converting data from one format to another</li>
</ul>
<p>While we data scientists love to create exciting data visualizations and insightful models (steps 3 and 4), usually much effort goes into obtaining and scrubbing the required data first (steps 1 and 2).
In <em>Data Jujitsu</em>, <span class="citation"><a href="#ref-Patil2012" role="doc-biblioref">DJ Patil</a><a href="#fn3" class="footnote-ref" id="fnref3"><sup>3</sup></a></span> states that “80% of the work in any data project is in cleaning the data.”
In <a href="#chapter-5-scrubbing-data">Chapter 5</a>, I demonstrate how the command line can help accomplish such data scrubbing operations.</p>
</div>
<div id="exploring-data" class="section level3" number="1.1.3">
<h3 number="1.1.3"><span class="header-section-number">1.1.3</span> Exploring Data</h3>
<p>Once you have scrubbed your data, you are ready to explore it.
This is where it gets interesting because when you’re exploring, you will truly get to know your data.
In <a href="#chapter-7-exploring-data">Chapter 7</a> I show you how the command line can be used to:</p>
<ul>
<li>Look at your data</li>
<li>Derive statistics from your data</li>
<li>Create insightful visualizations</li>
</ul>
<p>Command-line tools introduced in <a href="#chapter-7-exploring-data">Chapter 7</a> include: <code>csvstat</code><span class="citation"><a href="#fn4" class="footnote-ref" id="fnref4"><sup>4</sup></a></span> and <code>rush</code><span class="citation"><a href="#fn5" class="footnote-ref" id="fnref5"><sup>5</sup></a></span>.</p>
</div>
<div id="modeling-data" class="section level3" number="1.1.4">
<h3 number="1.1.4"><span class="header-section-number">1.1.4</span> Modeling Data</h3>
<p>If you want to explain the data or predict what will happen, you probably want to create a statistical model of your data.
Techniques to create a model include clustering, classification, regression, and dimensionality reduction.
The command line is not suitable for programming a new type of model from scratch.
It is, however, very useful to be able to build a model from the command line.
In <a href="#chapter-9-modeling-data">Chapter 9</a> I will introduce several command-line tools that either build a model locally or employ an API to perform the computation in the cloud.</p>
</div>
<div id="interpreting-data" class="section level3" number="1.1.5">
<h3 number="1.1.5"><span class="header-section-number">1.1.5</span> Interpreting Data</h3>
<p>The final and perhaps most important step in the OSEMN model is interpreting data.
This step involves:</p>
<ul>
<li>Drawing conclusions from your data</li>
<li>Evaluating what your results mean</li>
<li>Communicating your result</li>
</ul>
<p>To be honest, the computer is of little use here, and the command line does not really come into play at this stage.
Once you have reached this step, it’s up to you.
This is the only step in the OSEMN model which does not have its own chapter.
Instead, I refer you to the book <em>Thinking with Data</em> by <span class="citation"><a href="#ref-Shron2014" role="doc-biblioref">Max Shron</a><a href="#fn6" class="footnote-ref" id="fnref6"><sup>6</sup></a></span>.</p>
</div>
</div>
<div id="intermezzo-chapters" class="section level2" number="1.2">
<h2 number="1.2"><span class="header-section-number">1.2</span> Intermezzo Chapters</h2>
<p>Besides the chapters that cover the OSEMN steps, there are four intermezzo chapters.
Each discusses a more general topic concerning data science, and how the command line is employed for that.
These topics are applicable to any step in the data science process.</p>
<p>In <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>, I discuss how to create reusable tools for the command line.
These personal tools can come from both long commands that you have typed on the command line, or from existing code that you have written in, say, Python or R.
Being able to create your own tools allows you to become more efficient and productive.</p>
<p>Because the command line is an interactive environment for doing data science, it can become challenging to keep track of your workflow.
In <a href="#chapter-6-project-management-with-make">Chapter 6</a>, I demonstrate a command-line tool called <code>make</code>, which allows you to define your data science workflow in terms of tasks and the dependencies between them.
This tool increases the reproducibility of your workflow, not only for you but also for your colleagues and peers.</p>
<p>In <a href="#chapter-8-parallel-pipelines">Chapter 8</a>, I explain how your commands and tools can be sped up by running them in parallel.
Using a command-line tool called GNU Parallel<span class="citation"><a href="#fn7" class="footnote-ref" id="fnref7"><sup>7</sup></a></span>, you can apply command-line tools to very large datasets and run them on multiple cores or even remote machines.</p>
<p>In <a href="#chapter-10-polyglot-data-science">Chapter 10</a>, I discuss how to employ the power of the command line in other environments and programming languages such as R, RStudio, Python, Jupyter Notebooks, and even Apache Spark.</p>
</div>
<div id="what-is-the-command-line" class="section level2" number="1.3">
<h2 number="1.3"><span class="header-section-number">1.3</span> What is the Command Line?</h2>
<p>Before I discuss <em>why</em> you should use the command line for data science, let’s take a peek at <em>what</em> the command line actually looks like (it may be already familiar to you).
Figure @ref(fig:mac-terminal) and Figure @ref(fig:ubuntu-terminal) show a screenshot of the command line as it appears by default on macOS and Ubuntu, respectively.
Ubuntu is a particular distribution of GNU/Linux, and it’s the one I’ll be using in this book.</p>
<div class="figure" style="text-align: center">
<img src="images/screenshot_terminal_mac_dst.png" alt="Command line on macOS" width="90%" />
<p class="caption">
(#fig:mac-terminal)Command line on macOS
</p>
</div>
<div class="figure" style="text-align: center">
<img src="images/screenshot_terminal_ubuntu_dst.png" alt="Command line on Ubuntu" width="90%" />
<p class="caption">
(#fig:ubuntu-terminal)Command line on Ubuntu
</p>
</div>
<p>The window shown in the two screenshots is called the <em>terminal</em>.
This is the program that enables you to interact with the shell.
It is the shell that executes the commands I type in.
In <a href="#chapter-2-getting-started">Chapter 2</a>, I explain these two terms in more detail.</p>

<div class="rmdnote">
I’m not showing the Microsoft Windows command line (also known as the Command Prompt or PowerShell), because it’s fundamentally different and incompatible with the commands presented in this book.
The good news is that you can install a Docker image on Microsoft Windows so that you’re able to follow along.
How to install the Docker image is explained in <a href="#chapter-2-getting-started">Chapter 2</a>.
</div>
<p>Typing commands is a very different way of interacting with your computer than through a <em>graphical user interface</em> (GUI).
If you are mostly used to processing data in, say, Microsoft Excel, then this approach may seem intimidating at first.
Don’t be afraid.
Trust me when I say that you’ll get used to working at the command line very quickly.</p>
<p>In this book, the commands that I type and the output that they generate are displayed as text.
For example, the contents of the terminal in the two screenshots would look like this:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">whoami</span>
dst
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">date</span>
Tue Dec 14 11:43:30 AM CET 2021
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'The command line is awesome!'</span> | <span style="color: #5f8700">cowsay</span> -f tux
 ______________________________
&lt; The command line is awesome! &gt;
 ------------------------------
   \
    \
        .--.
       |o_o |
       |:_/ |
      //   \ \
     (|     | )
    /'\_   _/`\
    \___)=(___/
 
 
<span style="font-weight: bold">$</span></pre>
<p>You’ll also notice that each command is preceded with a dollar sign (<strong><code>$</code></strong>).
This is called the prompt.
The prompt in the two screenshots showed more information, namely the username, the date, and a penguin.
It’s a convention to show only a dollar sign in examples, because the prompt (1) can change during a session (when you go to a different directory), (2) can be customized by the user (e.g., it can also show the time or the current <code>git</code><span class="citation"><a href="#fn8" class="footnote-ref" id="fnref8"><sup>8</sup></a></span> branch you’re working on), and (3) is irrelevant for the commands themselves.</p>
<p>In the next chapter I’ll explain much more about essential command-line concepts.
Now it’s time to first explain <em>why</em> you should learn to use the command line for doing data science.</p>
</div>
<div id="why-data-science-at-the-command-line" class="section level2" number="1.4">
<h2 number="1.4"><span class="header-section-number">1.4</span> Why Data Science at the Command Line?</h2>
<p>The command line has many great advantages that can really make you a more efficient and productive data scientist.
Roughly grouping the advantages, the command line is: agile, augmenting, scalable, extensible, and ubiquitous.
I elaborate on each advantage below.</p>
<div id="the-command-line-is-agile" class="section level3" number="1.4.1">
<h3 number="1.4.1"><span class="header-section-number">1.4.1</span> The Command Line is Agile</h3>
<p>The first advantage of the command line is that it allows you to be agile.
Data science has a very interactive and exploratory nature, and the environment that you work in needs to allow for that.
The command line achieves this by two means.</p>
<p>First, the command line provides a so-called <em>read-eval-print-loop</em> (REPL).
This means that you type in command, press <strong><code>Enter</code></strong>, and the command is evaluated immediately.
A REPL is often much more convenient for doing data science than the edit-compile-run-debug cycle associated with scripts, large programs, and, say, Hadoop jobs.
Your commands are executed immediately, may be stopped at will, and can be changed quickly.
This short iteration cycle really allows you to play with your data.</p>
<p>Second, the command line is very close to the file system.
Because data is the main ingredient for doing data science, it is important to be able to work easily with the files that contain your dataset.
The command line offers many convenient tools for this.</p>
</div>
<div id="the-command-line-is-augmenting" class="section level3" number="1.4.2">
<h3 number="1.4.2"><span class="header-section-number">1.4.2</span> The Command Line is Augmenting</h3>
<p>The command line integrates well with other technologies.
Whatever technology your data science workflow currently includes (whether it’s R, Python, or Excel), please know that I’m not suggesting you abandon that workflow.
Instead, consider the command line as an augmenting technology that amplifies the technologies you’re currently employing.
It can do so in three ways.</p>
<p>First, the command line can act as a glue between many different data science tools.
One way to glue tools is by connecting the output from the first tool to the input of the second tool.
In <a href="#chapter-2-getting-started">Chapter 2</a> I explain how this works.</p>
<p>Second, you can often delegate tasks to the command line from your own environment.
For example, Python, R, and Apache Spark allow you to run command-line tools and capture their output.
I demonstrate this with examples in <a href="#chapter-10-polyglot-data-science">Chapter 10</a>.</p>
<p>Third, you can convert your code (e.g., a Python or R script) into a reusable command-line tool.
That way, it doesn’t matter anymore in what language it’s written.
Now, it can be used from the command line directly or from any environment that integrates with the command line as mentioned in the previous paragraph.
I explain how to this in <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>.</p>
<p>In the end, every technology has its strengths and weaknesses, so it’s good to know several and use whichever is most appropriate for the task at hand.
Sometimes that means using R, sometimes the command line, and sometimes even pen and paper.
By the end of this book you’ll have a solid understanding of when you should use the command line, and when you’re better off continuing with your favorite programming language or statistical computing environment.</p>
</div>
<div id="the-command-line-is-scalable" class="section level3" number="1.4.3">
<h3 number="1.4.3"><span class="header-section-number">1.4.3</span> The Command Line is Scalable</h3>
<p>As I’ve said before, working on the command line is very different from using a GUI.
On the command line you do things by typing, whereas with a GUI, you do things by pointing and clicking with a mouse.</p>
<p>Everything that you type manually on the command line can also be automated through scripts and tools.
This makes it very easy to re-run your commands in case you made a mistake, when the input data has changed, or because your colleague wants to perform the same analysis.
Moreover, your commands can be run at specific intervals, on a remote server, and in parallel on many chunks of data (more on that in <a href="#chapter-8-parallel-pipelines">Chapter 8</a>).</p>
<p>Because the command line is automatable, it becomes scalable and repeatable.
It’s not straightforward to automate pointing and clicking, which makes a GUI a less suitable environment for doing scalable and repeatable data science.</p>
</div>
<div id="the-command-line-is-extensible" class="section level3" number="1.4.4">
<h3 number="1.4.4"><span class="header-section-number">1.4.4</span> The Command Line is Extensible</h3>
<p>The command line itself was invented over 50 years ago.
Its core functionality has largely remained unchanged, but its <em>tools</em>, which are the workhorses of the command-line, are being developed on a daily basis.</p>
<p>The command line itself is language-agnostic.
This allows the command-line tools to be written in many different programming languages.
The open source community is producing many free and high-quality command-line tools that we can use for data science.</p>
<p>These command-line tools can work together, which makes the command line very flexible.
You can also create your own tools, allowing you to extending the effective functionality of the command line.</p>
</div>
<div id="the-command-line-is-ubiquitous" class="section level3" number="1.4.5">
<h3 number="1.4.5"><span class="header-section-number">1.4.5</span> The Command Line is Ubiquitous</h3>
<p>Because the command line comes with any Unix-like operating system, including Ubuntu Linux and macOS, it can be found in many places.
Plus, 100% of the top 500 supercomputers are running Linux.<a href="#fn9" class="footnote-ref" id="fnref9"><sup>9</sup></a>
So, if you ever get your hands on one of those supercomputers (or if you ever find yourself in Jurassic Park with the doorlocks not working), you better know your way around the command line!</p>
<p>But Linux not only runs on supercomputers.
It also runs on servers, laptops, and embedded systems.
These days, many companies offer cloud computing, where you can easily launch new machines on the fly.
If you ever log in to such a machine (or a server in general), it’s almost certain that you’ll arrive at the command line.</p>
<p>It’s also important to note that the command line isn’t just a hype.
This technology has been around for more than five decades, and I’m convinced that it’s here to stay for another five.
Learning how to use the command line (for data science and in general) is therefore a worthwhile investment.</p>
</div>
</div>
<div id="summary" class="section level2" number="1.5">
<h2 number="1.5"><span class="header-section-number">1.5</span> Summary</h2>
<p>In this chapter I have introduced you to OSEMN model for doing the data science, which I use as a guide throughout the book.
I have provided some background about the Unix command line and hopefully convinced you that it’s a suitable environment for doing data science.
In the next chapter I’m going to show you how to get started by installing the datasets and tools and explaining the fundamental concepts.</p>
</div>
<div id="for-further-exploration" class="section level2" number="1.6">
<h2 number="1.6"><span class="header-section-number">1.6</span> For Further Exploration</h2>
<ul>
<li>The book <em>UNIX: A History and a Memoir</em> by Brian W. Kernighan tells the story of Unix, explaining what it is, how it was developed, and why it matters.</li>
<li>In 2018 I gave a presentation titled <em>50 Reasons to Learn the Shell for Doing Data Science</em> at Strata London. You can <a href="https://datascienceatthecommandline.com/resources/50-reasons.pdf">read the slides</a> if you need even more convincing.</li>
<li>The short but sweet book <em>Thinking with Data</em> by Max Shron focuses on the <em>why</em> instead of the <em>how</em> and provides a framework for defining your data science project that will help you ask the right questions and solve the right problems.</li>
</ul>
<!--chapter:end:01.Rmd-->
</div>
</div>
<div id="chapter-2-getting-started" class="section level1" number="2">
<h1 number="2"><span class="header-section-number">2</span> Getting Started</h1>
<p>In this chapter I’m going to make sure that you have all the prerequisites for doing data science at the command line.
The prerequisites fall into three parts: (1) having the same data sets that I use in this book, (2) having a proper environment with all the command-line tools that I use throughout this book, and (3) understanding the essential concepts that come into play when using the command line.</p>
<p>First, I describe how to download the datasets.
Second, I explain how to install the Docker image, which is a virtual environment based on Ubuntu Linux that contains all the necessary command-line tools.
Subsequently, I go over the essential Unix concepts through examples.</p>
<p>By the end of this chapter, you’ll have everything you need in order to continue with the first step of doing data science, namely obtaining data.</p>
<div id="getting-the-data" class="section level2" number="2.1">
<h2 number="2.1"><span class="header-section-number">2.1</span> Getting the Data</h2>
<p>The datasets that I use in this book can be downloaded as follows:</p>
<ol style="list-style-type: decimal">
<li>Download the ZIP file from <a href="https://www.datascienceatthecommandline.com/2e/data.zip" class="uri">https://www.datascienceatthecommandline.com/2e/data.zip</a>.</li>
<li>Create a new directory. You can give this directory any name you want, but I recommend you stick to lowercase letters, numbers, and maybe a hyphen or underscore so that it’s easier to work with at the command line. For example: <em>dsatcl2e-data</em>. Remember where this directory is.</li>
<li>Move the ZIP file to that new directory and unpack it.</li>
<li>This directory now contains one subdirectory per chapter.</li>
</ol>
<p>In the next section I explain how to install the environment containing all the command-line tools to work with this data.</p>
</div>
<div id="docker-image" class="section level2" number="2.2">
<h2 number="2.2"><span class="header-section-number">2.2</span> Installing the Docker Image</h2>
<p>In this book we use many different command-line tools.
Unix often comes with a lot of command-line tools pre-installed and offers many packages that contain more relevant tools.
Installing these packages yourself is often not too difficult.
However, we’ll also use tools that are not available as packages and require a more manual, and more involved installation.
In order to acquire the necessary command-line tools without having to go through the installation process of each, I encourage you, whether you’re on Windows, macOS, or Linux, to install the Docker image that was created specifically for this book.</p>
<p>A Docker image is a bundle of one or more applications together with all their dependencies.
A Docker container is an isolated environment that runs an image.
You can manage Docker images and containers the <code>docker</code> command-line tool (which is what you’ll do below) or the Docker GUI.
In a way, a Docker container is like a virtual machine, only a Docker container uses far fewer resources.
At the end of this chapter I suggest some resources to learn more about Docker.</p>

<div class="rmdtip">
If you still prefer to run the command-line tools natively rather than inside a Docker container, then you can, of course, install the command-line tools individually yourself.
Please be aware that this is a very time-consuming process.
The Appendix lists all the command-line tools used in the book.
The installation instructions are for Ubuntu only.
The scripts and data sets used in the book can be obtained by cloning this book’s <a href="https://github.com/datasciencetoolbox/datasciencetoolbox">GitHub repository</a>.
</div>
<p>To install the Docker image, you first need to download and install Docker itself from <a href="https://www.docker.com/products/docker">the Docker website</a>.
Once Docker is installed, you invoke the following command on your terminal or command prompt to download the Docker image (don’t type the dollar sign):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">docker</span> pull datasciencetoolbox/dsatcl2e</pre>
<p>You can run the Docker image as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">docker</span> run --rm -it datasciencetoolbox/dsatcl2e</pre>
<p>You’re now inside an isolated environment—known as a <em>Docker container</em>—with all the necessary command-line tools installed.
If the following command produces an enthusiastic cow, then you know everything is working correctly:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cowsay</span> <span style="color: #af8700">"Let's moove</span><span style="color: #00afaf">\!</span><span style="color: #af8700">"</span>
 ______________
&lt; Let's moove! &gt;
 --------------
        \   ^__^
         \  (oo)\_______
            (__)\       )\/\
                ||----w |
                ||     ||</pre>
<p>If you want to get data in and out of the container, you can add a volume, which means that a local directory gets mapped to a directory inside the container.
I recommend that you first create a new directory, navigate to this new directory, and then run the following when you’re on macOS or Linux:</p>
<pre><span style="font-weight: bold">$</span> docker run --rm -it -v "$(p</pre>
<p>Or the following when you’re on Windows and using the Command Prompt (also known as <code>cmd</code>):</p>
<pre>C:\&gt; docker run --rm -it -v "%cd%":/data datasciencetoolbox/dsatcl2e</pre>
<p>Or the following when you’re using Windows PowerShell:</p>
<pre>PS C:\&gt; docker run --rm -it -v ${PWD}:/data datasciencetoolbox/dsatcl2e</pre>
<p>In the above commands, the option <code>-v</code> instructs <code>docker</code> to map the current directory to the <em>/data</em> directory inside the container, so this is the place to get data in and out of the Docker container.</p>

<div class="rmdnote">
If you would like to know more about the Docker image you can <a href="https://hub.docker.com/r/datasciencetoolbox/dsatcl2e">visit it on Docker Hub</a>.
</div>
<p>When you’re done, you can shut down the Docker container by typing <code>exit</code>.</p>
</div>
<div id="essential-concepts" class="section level2" number="2.3">
<h2 number="2.3"><span class="header-section-number">2.3</span> Essential Unix Concepts</h2>
<p>In <a href="#chapter-1-introduction">Chapter 1</a>, I briefly showed you what the command line is.
Now that you are running the Docker image, we can really get started.
In this section, I discuss several concepts and tools that you will need to know in order to feel comfortable doing data science at the command line.
If, up to now, you have been mainly working with graphical user interfaces, then this might be quite a change.
But don’t worry, I’ll start at the beginning, and very gradually go to more advanced topics.</p>

<div class="rmdnote">
This section is not a complete course in Unix.
I will only explain the concepts and tools that are relevant for to doing data science.
One of the advantages of the Docker image is that a lot is already set up.
If you wish to know more, consult the Further Reading Section at the end of this chapter.
</div>
<div id="the-environment" class="section level3" number="2.3.1">
<h3 number="2.3.1"><span class="header-section-number">2.3.1</span> The Environment</h3>
<p>So you’ve just logged into a brand new environment.
Before you do anything, it’s worthwhile to get a high-level understanding of this environment.
It’s roughly defined by four layers, which I briefly discuss from the top down.</p>
<dl>
<dt>Command-line tools</dt>
<dd><p>First and foremost, there are the command-line tools that you work with.
We use them by typing their corresponding commands.
There are different types of command-line tools, which I will discuss in the next section.
Examples of tools are: <code>ls</code><span class="citation"><a href="#fn10" class="footnote-ref" id="fnref10"><sup>10</sup></a></span>, <code>cat</code><span class="citation"><a href="#fn11" class="footnote-ref" id="fnref11"><sup>11</sup></a></span>, and <code>jq</code><span class="citation"><a href="#fn12" class="footnote-ref" id="fnref12"><sup>12</sup></a></span>.</p>
</dd>
<dt>Terminal</dt>
<dd><p>The terminal, which is the second layer, is the application where we type our commands in. If you see the following text mentioned in the book:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 3
1
2
3</pre>
<p>then you would type <code>seq 3</code> into your terminal and press <strong><code>Enter</code></strong>.
(The command-line tool <code>seq</code><span class="citation"><a href="#fn13" class="footnote-ref" id="fnref13"><sup>13</sup></a></span>, as you can see, generates a sequence of numbers.) You do not type the dollar sign.
It’s just there to tell you that this a command you can type in the terminal.
This dollar sign is known as the prompt.
The text below <code>seq 3</code> is the output of the command.</p>
</dd>
<dt>Shell</dt>
<dd><p>The third layer is the shell. Once we have typed in our command and pressed <strong><code>Enter</code></strong>, the terminal sends that command to the shell. The <em>shell</em> is a program that interprets the command. I use the Z shell, but there are many others available such as Bash and Fish.</p>
</dd>
<dt>Operating system</dt>
<dd><p>The fourth layer is the operating system, which is GNU/Linux in our case. Linux is the name of the kernel, which is the heart of the operating system. The kernel is in direct contact with the CPU, disks, and other hardware. The kernel also executes our command-line tools. GNU, which stands for GNU’s not UNIX, refers to the set of basic tools. The Docker image is based on a particular GNU/Linux distribution called Ubuntu.</p>
</dd>
</dl>
</div>
<div id="executing-a-command-line-tool" class="section level3" number="2.3.2">
<h3 number="2.3.2"><span class="header-section-number">2.3.2</span> Executing a Command-line Tool</h3>
<p>Now that you have a basic understanding of the environment, it is high time that you try out some commands.
Type the following in your terminal (without the dollar sign) and press <strong><code>Enter</code></strong>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">pwd</span>
/home/dst</pre>
<p>You just executed a command that contained a single command-line tool.
The tool <code>pwd</code><span class="citation"><a href="#fn14" class="footnote-ref" id="fnref14"><sup>14</sup></a></span> outputs the name of the directory where you currently are.
By default, when you login, this is your home directory.</p>
<p>The command-line tool <code>cd</code>, which is a Z shell builtin, allows you to navigate to a different directory:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch02</span> <span class="callout">&#10122;</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">pwd</span> <span class="callout">&#10123;</span>
/data/ch02
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">..</span> <span class="callout">&#10124;</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">pwd</span> <span class="callout">&#10125;</span>
/data
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">ch02</span> <span class="callout">&#10126;</span></pre>
<p><span class="callout">&#10122;</span> Navigate to the directory <em>/data/ch02</em>.
<br><span class="callout">&#10123;</span> Print the current directory.
<br><span class="callout">&#10124;</span> Navigate to the parent directory.
<br><span class="callout">&#10125;</span> Print the current directory again.
<br><span class="callout">&#10126;</span> Navigate to the subdirectory <em>ch02</em>.</p>
<p>The part after <code>cd</code> specifies to which directory you want to navigate to.
Values that come after the command are called <em>command-line arguments</em> or <em>options</em>.
Two dots refer to the parent directory.
One dot, by the way, refers to the current directory.
While <code>cd .</code> wouldn’t have any effect, you’ll still see one dot being used in other places.
Let’s try a different command:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">head</span> -n 3 <span style="text-decoration: underline">movies.txt</span>
Matrix
Star Wars
Home Alone</pre>
<p>Here we pass three command-line arguments to <code>head</code><span class="citation"><a href="#fn15" class="footnote-ref" id="fnref15"><sup>15</sup></a></span>.
The first one is an option.
Here I used the short option <code>-n</code>.
Sometimes a short option has a long variant, which would be <code>--lines</code> in this case.
The second one is a value that belongs to the option.
The third one is a filename.
This particular command outputs the first three lines of file <em>/data/ch02/movies.txt</em>.</p>
</div>
<div id="five-types-of-command-line-tools" class="section level3" number="2.3.3">
<h3 number="2.3.3"><span class="header-section-number">2.3.3</span> Five Types of Command-line Tools</h3>
<p>I use the term <em>command-line tool</em> a lot, but so far, I haven’t yet explained what I actually mean by it.
I use it as an umbrella term for <em>anything</em> that can be executed from the command line (see Figure @ref(fig:umbrella)).
Under the hood, each command-line tool is one of the following five types:</p>
<ul>
<li>A binary executable</li>
<li>A shell builtin</li>
<li>An interpreted script</li>
<li>A shell function</li>
<li>An alias</li>
</ul>
<div class="figure" style="text-align: center">
<img src="images/dscl_0201.png" alt="I use the term command-line tool as an umbrella term" width="90%" />
<p class="caption">
(#fig:umbrella)I use the term command-line tool as an umbrella term
</p>
</div>
<p>It’s good to know the difference between the types.
The command-line tools that come pre-installed with the Docker image mostly comprise of the first two types (binary executable and shell builtin).
The other three types (interpreted script, shell function, and alias) allow us to further build up our data science toolbox and become more efficient and more productive data scientists.</p>
<dl>
<dt>Binary Executable</dt>
<dd><p>Binary executables are programs in the classical sense. A binary executable is created by compiling source code to machine code. This means that when you open the file in a text editor you cannot read it.</p>
</dd>
<dt>Shell Builtin</dt>
<dd><p>Shell builtins are command-line tools provided by the shell, which is the Z shell (or <code>zsh</code>) in our case. Examples include <code>cd</code> and <code>pwd</code>. Shell builtins may differ between shells. Like binary executables, they cannot be easily inspected or changed.</p>
</dd>
<dt>Interpreted Script</dt>
<dd><p>An interpreted script is a text file that is executed by a binary executable. Examples include: Python, R, and Bash scripts. One great advantage of an interpreted script is that you can read and change it. The script below is interpreted by Python not because of the file extension <em>.py</em>, but because the first line of the script defines the binary that should execute it.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">fac.py</span>
───────┬──────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">fac.py</span>
───────┼──────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env python</span>
   2   │
   3   │ <span style="color: #af005f">def</span> <span style="color: #0087ff">factorial</span>(x):
   4   │     result <span style="color: #af005f">=</span> <span style="color: #af8700">1</span>
   5   │     <span style="color: #af005f">for</span> i <span style="color: #af005f">in</span> <span style="color: #00afaf">range</span>(<span style="color: #af8700">2</span>, x <span style="color: #af005f">+</span> <span style="color: #af8700">1</span>):
   6   │         result <span style="color: #af005f">*=</span> i
   7   │     <span style="color: #af005f">return</span> result
   8   │
   9   │ <span style="color: #af005f">if</span> __name__ <span style="color: #af005f">==</span> <span style="color: #5f8700">"__main__"</span>:
  10   │     <span style="color: #af005f">import</span> sys
  11   │     x <span style="color: #af005f">=</span> int(sys.argv[<span style="color: #af8700">1</span>])
  12   │     sys.stdout.write(<span style="color: #af005f">f</span><span style="color: #5f8700">"</span>{factorial(x)}<span style="color: #00afaf">\n</span><span style="color: #5f8700">"</span>)
───────┴──────────────────────────────────────────────────────────────</pre>
<p>This script computes the factorial of the integer that we pass as a parameter. It can be invoked from the command line as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">./fac.py</span> 5
120</pre>
<p>In <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>, I’ll discuss in great detail how to create reusable command-line tools using interpreted scripts.</p>
</dd>
<dt>Shell Function</dt>
<dd><p>A shell function is a function that is, in our case, executed by <code>zsh</code>. They provide similar functionality to a script, but they are usually (but not necessarily) smaller than scripts. They also tend to be more personal. The following command defines a function called <code>fac</code>, which, just like the interpreted Python script above, computes the factorial of the integer we pass as a parameter. It does by generating a list of numbers using <code>seq</code>, putting those numbers on one line with <code>*</code> as the delimiter using <code>paste</code><span class="citation"><a href="#fn16" class="footnote-ref" id="fnref16"><sup>16</sup></a></span>, and passing this equation into <code>bc</code><span class="citation"><a href="#fn17" class="footnote-ref" id="fnref17"><sup>17</sup></a></span>, which evaluates it and outputs the result.</p>
<pre><span style="font-weight: bold">$</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">fac</span><span style="color: #af8700"></span><span style="color: #af8700">()</span> <span style="color: #af8700">{</span> <span style="color: #af8700">(</span><span style="color: #5f8700">echo</span> 1; <span style="color: #5f8700">seq</span> $1<span style="color: #af8700">)</span> | <span style="color: #5f8700">paste</span> -s -d\* - | <span style="color: #5f8700">bc</span>; <span style="color: #af8700">}</span>

<span style="font-weight: bold">$</span> <span style="color: #5f8700">fac</span> 5
120</pre>
<p>The file <em>~/.zshrc</em>, which is a configuration file for Z shell, is a good place to define your shell functions, so that they are always available.</p>
</dd>
<dt>Alias</dt>
<dd><p>Aliases are like macros. If you often find yourself executing a certain command with the same parameters (or a part of it), you can define an alias for it to save time. Aliases are also very useful when you continue to misspell a certain command (Chris Wiggins maintains a <a href="https://github.com/chrishwiggins/mise/blob/master/sh/aliases-public.sh">useful list of aliases</a>). The following command defines such an alias:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">alias</span> l=<span style="color: #af8700">'ls --color -lhF --group-directories-first'</span>

<span style="font-weight: bold">$</span> <span style="color: #5f8700">alias</span> les=less</pre>
<p>Now, if you type the following on the command line, the shell will replace each alias it finds with its value:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data</span>

<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 40K
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch02</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch03</span>/
drwxr-xr-x 3 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch04</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch05</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch06</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch07</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch08</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch09</span>/
drwxr-xr-x 4 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch10</span>/
drwxr-xr-x 3 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">csvconf</span>/

<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">ch02</span></pre>
<p>Aliases are simpler than shell functions as they don’t allow parameters. The function <code>fac</code> could not have been defined using an alias because of the parameter. Still, aliases allow you to save lots of keystrokes. Like shell functions, aliases are often defined in the file <em>.zshrc</em>, which is located in your home directory. To see all aliases currently defined, you run <code>alias</code> without arguments. Try it. What do you see?</p>
</dd>
</dl>
<p>In this book I’ll focus mostly on the last three types of command-line tools: interpreted scripts, shell functions, and aliases.
This is because these can easily be changed.
The purpose of a command-line tool is to make your life on the easier, and to make you a more productive and more efficient data scientist.
You can find out the type of a command-line tool with <code>type</code> (which is itself a shell builtin):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> -a pwd
pwd is a shell builtin
pwd is /usr/bin/pwd
pwd is /bin/pwd
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> -a cd
cd is a shell builtin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> -a fac
fac is a shell function
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> -a l
l is an alias for ls --color -lhF --group-directories-first</pre>
<p><code>type</code> returns three command-line tools for <code>pwd</code>.
In that case, the first reported command-line tool is used when you type <code>pwd</code>.
In the next section we’ll look at how to combine command-line tools.</p>
</div>
<div id="combining-command-line-tools" class="section level3" number="2.3.4">
<h3 number="2.3.4"><span class="header-section-number">2.3.4</span> Combining Command-line Tools</h3>
<p>Because most command-line tools adhere to the Unix philosophy<span class="citation"><a href="#fn18" class="footnote-ref" id="fnref18"><sup>18</sup></a></span>, they are designed to do only thing, and do it really well.
For example, the command-line tool <code>grep</code><span class="citation"><a href="#fn19" class="footnote-ref" id="fnref19"><sup>19</sup></a></span> can filter lines, <code>wc</code><span class="citation"><a href="#fn20" class="footnote-ref" id="fnref20"><sup>20</sup></a></span> can count lines, and <code>sort</code><span class="citation"><a href="#fn21" class="footnote-ref" id="fnref21"><sup>21</sup></a></span> can sort lines.
The power of the command line comes from its ability to combine these small, yet powerful command-line tools.</p>
<p>This power is made possible by managing the communication streams of these tools.
Each tool has three standard communication streams: standard input, standard output, and standard error.
These are often abbreviated as <em><code>stdin</code></em>, <em><code>stdout</code></em>, and <em><code>stderr</code></em>.</p>
<p>Both the standard output and standard error are, by default, redirected to the terminal, so that both normal output and any error messages are printed on the screen.
Figure @ref(fig:diagram-essential-streams) illustrates this for both <code>pwd</code> and <code>rev</code><span class="citation"><a href="#fn22" class="footnote-ref" id="fnref22"><sup>22</sup></a></span>.
If you run <code>rev</code>, you’ll see that nothing happens.
That’s because <code>rev</code> expects input, and by the default, that’s any keys pressed on the keyboard.
Try typing a sentence and press <strong><code>Enter</code></strong>.
<code>rev</code> immediately responds with your input in reverse.
You can stop sending input by pressing <strong><code>Ctrl-D</code></strong> after which <code>rev</code> will stop.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0202.png" alt="Every tool has three standard streams: standard input (*`stdin`*), standard output (*`stdout`*), and standard error (*`stderr`*)" width="90%" />
<p class="caption">
(#fig:diagram-essential-streams)Every tool has three standard streams: standard input (<em><code>stdin</code></em>), standard output (<em><code>stdout</code></em>), and standard error (<em><code>stderr</code></em>)
</p>
</div>
<p>In practice, you’ll not use the keyboard as a source of input, but the output generated by other tools and the contents of files.
For example, with <code>curl</code> we can download the book <em>Alice’s Adventures in Wonderland</em> by Lewis Carrol and <em>pipe</em> that to the next tool.
(I’ll discuss <code>curl</code> in more detail in <a href="#chapter-3-obtaining-data">Chapter 3</a>.)
This is done using the pipe operator (<code>|</code>).</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0203.png" alt="The output from a tool can be piped to another tool" width="90%" />
<p class="caption">
(#fig:diagram-essential-pipe)The output from a tool can be piped to another tool
</p>
</div>
<p>We can <em>pipe</em> the output of <code>curl</code> to <code>grep</code> to filter lines on a pattern.
Imagine that we want to see the chapters listed in the table of contents:
We can combine <code>curl</code> and <code>grep</code> as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> | <span style="color: #5f8700">grep</span> <span style="color: #af8700">" CHAPTER"</span>
 CHAPTER I.     Down the Rabbit-Hole
 CHAPTER II.    The Pool of Tears
 CHAPTER III.   A Caucus-Race and a Long Tale
 CHAPTER IV.    The Rabbit Sends in a Little Bill
 CHAPTER V.     Advice from a Caterpillar
 CHAPTER VI.    Pig and Pepper
 CHAPTER VII.   A Mad Tea-Party
 CHAPTER VIII.  The Queen’s Croquet-Ground
 CHAPTER IX.    The Mock Turtle’s Story
 CHAPTER X.     The Lobster Quadrille
 CHAPTER XI.    Who Stole the Tarts?
 CHAPTER XII.   Alice’s Evidence</pre>
<p>And if we wanted to know <em>how many</em> chapters the book has, we can use <code>wc</code>, which is very good at counting things:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">grep</span> <span style="color: #af8700">" CHAPTER"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">wc</span> -l <span class="callout">&#10122;</span>
12</pre>
<p><span class="callout">&#10122;</span> The option <code>-l</code> specifies that <code>wc</code> should only output the number of lines that are pass into it. By default it also returns the number of characters and words.</p>
<p>You can think of piping as an automated copy and paste.
Once you get the hang of combining tools using the pipe operator, you’ll find that there are virtually no limits to this.</p>
</div>
<div id="redirecting-input-and-output" class="section level3" number="2.3.5">
<h3 number="2.3.5"><span class="header-section-number">2.3.5</span> Redirecting Input and Output</h3>
<p>Besides piping the output from one tool to another tool, you can also save it to a file.
The file will be saved it in the current directory, unless a full path is given.
This is called <em>output redirection</em>, and works as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> <span style="color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> | <span style="color: #5f8700">grep</span> <span style="color: #af8700">" CHAPTER"</span> <span style="color: #af8700">&gt;</span> chapter
s.txt
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  170k  100  170k    0     0   202k      0 --:--:-- --:--:-- --:--:--  203k
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">chapters.txt</span>
 CHAPTER I.     Down the Rabbit-Hole
 CHAPTER II.    The Pool of Tears
 CHAPTER III.   A Caucus-Race and a Long Tale
 CHAPTER IV.    The Rabbit Sends in a Little Bill
 CHAPTER V.     Advice from a Caterpillar
 CHAPTER VI.    Pig and Pepper
 CHAPTER VII.   A Mad Tea-Party
 CHAPTER VIII.  The Queen’s Croquet-Ground
 CHAPTER IX.    The Mock Turtle’s Story
 CHAPTER X.     The Lobster Quadrille
 CHAPTER XI.    Who Stole the Tarts?
 CHAPTER XII.   Alice’s Evidence</pre>
<p>Here, we save the output of <code>grep</code> to a file named <em>chapters.txt</em> in the directory <em>/data/ch02</em>.
If this file does not exist yet, it will be created.
If this file already exists, its contents are overwritten.
Figure @ref(fig:diagram-essential-redirect-stdout) illustrates how output redirection works conceptually.
Note that the standard error is still redirected to the terminal.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0204.png" alt="The output from a tool can be redirected to a file" width="90%" />
<p class="caption">
(#fig:diagram-essential-redirect-stdout)The output from a tool can be redirected to a file
</p>
</div>
<p>You can also append the output to a file with <code>&gt;&gt;</code>, meaning the output is added after the original contents:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> -n <span style="color: #af8700">"Hello"</span> <span style="color: #af8700">&gt;</span> greeting.txt
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">" World"</span> <span style="color: #af8700">&gt;&gt;</span> <span style="text-decoration: underline">greeting.txt</span></pre>
<p>The tool <code>echo</code> outputs the value you specify.
The <code>-n</code> option, which stands for <em>newline</em>, specifies that <code>echo</code> should not output a trailing newline.</p>
<p>Saving the output to a file is useful if you need to store intermediate results, for example to continue with your analysis at a later stage.
To use the contents of the file <em>greeting.txt</em> again, we can use <code>cat</code>, which reads a file prints it.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">greeting.txt</span>
Hello World
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">greeting.txt</span> | <span style="color: #5f8700">wc</span> -w <span class="callout">&#10122;</span>
2</pre>
<p><span class="callout">&#10122;</span> The <code>-w</code> option indicates <code>wc</code> to only count words.</p>
<p>The same result can be achieved by using the smaller-than-sign (<code>&lt;</code>):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">greeting.txt</span> <span style="color: #5f8700">wc</span> -w
2</pre>
<p>This way, you are directly passing the file to the standard input of <code>wc</code> without running an additional process<a href="#fn23" class="footnote-ref" id="fnref23"><sup>23</sup></a>.
Figure @ref(fig:diagram-essential-stdin-cat) illustrates how these two ways work.
Again, the final output is the same.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0205.png" alt="Two ways to use the contents of a file as input" width="90%" />
<p class="caption">
(#fig:diagram-essential-stdin-cat)Two ways to use the contents of a file as input
</p>
</div>
<p>Like many command-line tools, <code>wc</code> allows one or more filenames to be specified as arguments.
For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">wc</span> -w <span style="text-decoration: underline">greeting.txt</span> <span style="text-decoration: underline">movies.txt</span>
 2 greeting.txt
11 movies.txt
13 total</pre>
<p>Note that in this case, <code>wc</code> also outputs the name of the files.</p>
<p>You can suppress the output of any tool by redirecting it to a special file called <em>/dev/null</em>.
I often do this to suppress error messages (see Figure @ref(fig:diagram-essential-redirect-devnull) for an illustration).
The following causes <code>cat</code> to produce an error message because it cannot find the file <em>404.txt</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">movies.txt</span> 404.txt
Matrix
Star Wars
Home Alone
Indiana Jones
Back to the Future
/usr/bin/cat: 404.txt: No such file or directory</pre>
<p>You can redirect standard error to <em>/dev/null</em> as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">movies.txt</span> 404.txt <span style="color: #af8700">2&gt;</span> <span style="text-decoration: underline">/dev/null</span> <span class="callout">&#10122;</span>
Matrix
Star Wars
Home Alone
Indiana Jones
Back to the Future</pre>
<p><span class="callout">&#10122;</span> The <em><code>2</code></em> refers to standard error.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0206.png" alt="Redirecting *`stderr`* to */dev/null*" width="50%" />
<p class="caption">
(#fig:diagram-essential-redirect-devnull)Redirecting <em><code>stderr</code></em> to <em>/dev/null</em>
</p>
</div>
<p>Be careful not to read from and write to the same file.
If you do, you’ll end up with an empty file.
That’s because the tool of which the output is redirected, immediately opens that file for writing, and thereby emptying it.
There are two workarounds for this: (1) write to a different file and rename it afterwards with <code>mv</code> or (2) use <code>sponge</code><span class="citation"><a href="#fn24" class="footnote-ref" id="fnref24"><sup>24</sup></a></span>, which soaks up all its input before writing to a file.
Figure @ref(fig:diagram-essential-sponge) illustrates how this works.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0207.png" alt="Unless you use `sponge`, you cannot read from and write to the same file in one pipeline" width="90%" />
<p class="caption">
(#fig:diagram-essential-sponge)Unless you use <code>sponge</code>, you cannot read from and write to the same file in one pipeline
</p>
</div>
<p>For example, imagine you have used <code>dseq</code><span class="citation"><a href="#fn25" class="footnote-ref" id="fnref25"><sup>25</sup></a></span> to generate a file <em>dates.txt</em> and now you’d like to add line numbers using <code>nl</code><span class="citation"><a href="#fn26" class="footnote-ref" id="fnref26"><sup>26</sup></a></span>.
If you run the following, the file <em>dates.txt</em> will end up empty.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">dseq</span> 5 <span style="color: #af8700">&gt;</span> dates.txt
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">dates.txt</span> <span style="color: #5f8700">nl</span> <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">dates.txt</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">dates.txt</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">dates.txt</span>   &lt;EMPTY&gt;
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Instead, you can use one of the workarounds I just described:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">dseq</span> 5 <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">dates.txt
</span>
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">dates.txt</span> <span style="color: #5f8700">nl</span> <span style="color: #af8700">&gt;</span> dates-nl.txt
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">dates-nl.txt</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">dates-nl.txt</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │      1  2021-12-15
   2   │      2  2021-12-16
   3   │      3  2021-12-17
   4   │      4  2021-12-18
   5   │      5  2021-12-19
───────┴────────────────────────────────────────────────────────────────────────
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">dseq</span> 5 <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">dates.txt
</span>
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">dates.txt</span> <span style="color: #5f8700">nl</span> | <span style="color: #5f8700">sponge</span> <span style="text-decoration: underline">dates.txt</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">dates.txt</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">dates.txt</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │      1  2021-12-15
   2   │      2  2021-12-16
   3   │      3  2021-12-17
   4   │      4  2021-12-18
   5   │      5  2021-12-19
───────┴────────────────────────────────────────────────────────────────────────</pre>
</div>
<div id="working-with-files-and-directories" class="section level3" number="2.3.6">
<h3 number="2.3.6"><span class="header-section-number">2.3.6</span> Working With Files and Directories</h3>
<p>As data scientists, we work with a lot of data.
This data is often stored in files.
It is important to know how to work with files (and the directories they live in) on the command line.
Every action that you can do using a GUI, you can do with command-line tools (and much more).
In this section I introduce the most important ones to list, create, move, copy, rename, and delete files and directories.</p>
<p>Listing the contents of a directory can be done with <code>ls</code>.
If you don’t specify a directory, it lists the contents of the current directory.
I prefer <code>ls</code> to have a long listing format and the directories grouped before files.
Instead of typing the corresponding options each time, I use the alias <code>l</code>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">ls</span> <span style="text-decoration: underline">/data/ch10</span>
alice.txt  count.py  count.R  __pycache__  Untitled1337.ipynb
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">alias</span> l
l='ls --color -lhF --group-directories-first'
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span> <span style="text-decoration: underline">/data/ch10</span>
total 180K
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">__pycache__</span>/
-rw-r--r-- 1 dst dst 164K Dec 14 11:43 alice.txt
-rwxr--r-- 1 dst dst  408 Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">count.py</span>*
-rw-r--r-- 1 dst dst  460 Dec 14 11:43 count.R
-rw-r--r-- 1 dst dst 1.7K Dec 14 11:43 Untitled1337.ipynb</pre>
<p>You have already seen how we can create new files by redirecting the output with either <code>&gt;</code> or <code>&gt;&gt;</code>.
If you need to move a file to a different directory you can use <code>mv</code><span class="citation"><a href="#fn27" class="footnote-ref" id="fnref27"><sup>27</sup></a></span>:</p>
<pre>$ mv hello.txt /data/ch02</pre>
<p>You can also rename files with <code>mv</code>:</p>
<pre>$ cd data
$ mv hello.txt bye.txt</pre>
<p>You can also rename or move entire directories.
If you no longer need a file, you delete (or remove) it with <code>rm</code><span class="citation"><a href="#fn28" class="footnote-ref" id="fnref28"><sup>28</sup></a></span>:</p>
<pre>$ rm bye.txt</pre>
<p>If you want to remove an entire directory with all its contents, specify the <code>-r</code> option, which stands for recursive:</p>
<pre>$ rm -r /data/ch02/old</pre>
<p>If you want to copy a file, use <code>cp</code><span class="citation"><a href="#fn29" class="footnote-ref" id="fnref29"><sup>29</sup></a></span>.
This is useful for creating backups:</p>
<pre>$ cp server.log server.log.bak</pre>
<p>You can create directories using <code>mkdir</code><span class="citation"><a href="#fn30" class="footnote-ref" id="fnref30"><sup>30</sup></a></span>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">mkdir</span> logs
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 44K
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch02</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch03</span>/
drwxr-xr-x 3 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch04</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch05</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch06</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch07</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch08</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch09</span>/
drwxr-xr-x 4 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">ch10</span>/
drwxr-xr-x 3 dst dst 4.0K Dec 14 11:43 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">csvconf</span>/
drwxr-xr-x 2 dst dst 4.0K Dec 14 11:44 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">logs</span>/</pre>

<div class="rmdtip">
Using the command-line tools to manage your files can be scary at first, because you have no graphical overview of the file system to provide immediate feedback.
There are a few visual file managers that can help with this, such as GNU Midnight Commander, Ranger, and Vifm.
These are not installed in the Docker image, but you can install one yourself by running <code>sudo apt install</code> followed by either <code>mc</code>, <code>ranger</code>, or <code>vifm</code>.
</div>
<p>All of the above command-line tools accept the <code>-v</code> option, which stands for verbose, so that they output what’s going on.
For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">mkdir</span> -v backup
/usr/bin/mkdir: created directory 'backup'
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> -v <span style="color: #0087ff">*</span> <span style="text-decoration: underline">backup</span>
/usr/bin/cp: -r not specified; omitting directory 'backup'
/usr/bin/cp: -r not specified; omitting directory 'ch02'
/usr/bin/cp: -r not specified; omitting directory 'ch03'
/usr/bin/cp: -r not specified; omitting directory 'ch04'
/usr/bin/cp: -r not specified; omitting directory 'ch05'
/usr/bin/cp: -r not specified; omitting directory 'ch06'
/usr/bin/cp: -r not specified; omitting directory 'ch07'
/usr/bin/cp: -r not specified; omitting directory 'ch08'
/usr/bin/cp: -r not specified; omitting directory 'ch09'
/usr/bin/cp: -r not specified; omitting directory 'ch10'
/usr/bin/cp: -r not specified; omitting directory 'csvconf'
/usr/bin/cp: -r not specified; omitting directory 'logs'</pre>
<p>All tools but <code>mkdir</code> also accept the <code>-i</code> option, which stands for interactive, and causes the tools to ask you for confirmation.
For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rm</span> -i <span style="color: #0087ff">*</span>
zsh: sure you want to delete all 12 files in /data [yn]? n</pre>
</div>
<div id="managing-output" class="section level3" number="2.3.7">
<h3 number="2.3.7"><span class="header-section-number">2.3.7</span> Managing Output</h3>
<p>Sometimes a tools or sequence of tools produces too much output to include in the book.
Instead of manually altering such output, I prefer to be transparent by piping it through a helper tool.
You don’t necessarily have to do this, especially if you’re interested in the complete output.</p>
<p>Here are the tools that I use for making output manageable:</p>
<p>If often use <code>trim</code> to limit the output to a given height and width.
By default, output is trimmed to 10 lines and the width of the terminal.
Pass a negative number to disable trimming the height and or width.
For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">/data/ch07/tips.csv</span> | <span style="color: #5f8700">trim</span> 5 25
bill,tip,sex,smoker,day,…
16.99,1.01,Female,No,Sun…
10.34,1.66,Male,No,Sun,D…
21.01,3.5,Male,No,Sun,Di…
23.68,3.31,Male,No,Sun,D…
… with 240 more lines</pre>
<p>Other tools that I use to massage the output are: <code>head</code>, <code>tail</code>, <code>fold</code>, <code>paste</code>, and <code>column</code>.
The appendix contains examples for each of these.</p>
<p>If the output is a comma-separated values, I often pipe it through<code>csvlook</code> to turn it into a nice-looking table.
If you run <code>csvlook</code>, you’ll see the complete table.
I have redefined <code>csvlook</code> such that the table is shortened by <code>trim</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">which</span> csvlook
csvlook () {
        /usr/bin/csvlook "$@" | trim | sed 's/- | -/──┼──/g;s/| -/├──/g;s/- |/──
┤/;s/|/│/g;2s/-/─/g'
}
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> <span style="text-decoration: underline">/data/ch07/tips.csv</span>
│  bill │   tip │ sex    │ smoker │ day  │ time   │ size │
├───────┼───────┼────────┼────────┼──────┼────────┼──────┤
│ 16.99 │  1.01 │ Female │  False │ Sun  │ Dinner │    2 │
│ 10.34 │  1.66 │ Male   │  False │ Sun  │ Dinner │    3 │
│ 21.01 │  3.50 │ Male   │  False │ Sun  │ Dinner │    3 │
│ 23.68 │  3.31 │ Male   │  False │ Sun  │ Dinner │    2 │
│ 24.59 │  3.61 │ Female │  False │ Sun  │ Dinner │    4 │
│ 25.29 │  4.71 │ Male   │  False │ Sun  │ Dinner │    4 │
│  8.77 │  2.00 │ Male   │  False │ Sun  │ Dinner │    2 │
│ 26.88 │  3.12 │ Male   │  False │ Sun  │ Dinner │    4 │
… with 236 more lines</pre>
<p>I use <code>bat</code> to show the contents of a file where line numbers and syntax highlighting matters.
For example source code:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">/data/ch04/stream.py</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">/data/ch04/stream.py</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env python</span>
   2   │ <span style="color: #af005f">from</span> sys <span style="color: #af005f">import</span> stdin, stdout
   3   │ <span style="color: #af005f">while</span> <span style="color: #af8700">True</span>:
   4   │     line <span style="color: #af005f">=</span> stdin.readline()
   5   │     <span style="color: #af005f">if</span> <span style="color: #af005f">not</span> line:
   6   │         <span style="color: #af005f">break</span>
   7   │     stdout.write(<span style="color: #5f8700">"</span><span style="color: #af8700">%d</span><span style="color: #00afaf">\n</span><span style="color: #5f8700">"</span> <span style="color: #af005f">%</span> int(line)<span style="color: #af005f">**</span><span style="color: #af8700">2</span>)
   8   │     stdout.flush()
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Sometimes I add the <code>-A</code> option when I want to explicitly point out the spaces, tabs, and newlines in a file.</p>
<p>Sometimes it’s useful to write intermediate output to a file.
This allows you to inspect any step in your pipeline once it has completed.
You can insert the tool <code>tee</code> as often as you like in your pipeline.
I often use it to inspect a portion of the final output, while writing the complete output to file (see Figure @ref(fig:diagram-essential-tee).
Here, the complete output is written to <em>even.txt</em> and the first 5 lines are printed using <code>trim</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 0 2 100 | <span style="color: #5f8700">tee</span> even.txt | <span style="color: #5f8700">trim</span> 5
0
2
4
6
8
… with 46 more lines</pre>
<div class="figure" style="text-align: center">
<img src="images/dscl_0208.png" alt="With `tee`, you can write intermediate output to a file" width="90%" />
<p class="caption">
(#fig:diagram-essential-tee)With <code>tee</code>, you can write intermediate output to a file
</p>
</div>
<p>Lastly, to insert images that have been generated by command-line tools (so every image except screenshots and diagrams) I use <code>display</code>.
If you run <code>display</code> you’ll find that it doesn’t work.
In <a href="#chapter-7-exploring-data">Chapter 7</a>, I explain four options for you to display generated images from the command line.</p>
</div>
<div id="help" class="section level3" number="2.3.8">
<h3 number="2.3.8"><span class="header-section-number">2.3.8</span> Help!</h3>
<p>As you’re finding your way around the command-line, it may happen that you need help.
Even the most seasoned users need help at some point.
It is impossible to remember all the different command-line tools and their possible arguments.
Fortunately, the command line offers severals ways to get help.</p>
<p>The most important command to get help is perhaps <code>man</code><span class="citation"><a href="#fn31" class="footnote-ref" id="fnref31"><sup>31</sup></a></span>, which is short for <em>manual</em>.
It contains information for most command-line tools.
In case I forgot the options to the tool <code>tar</code>, which happens all the time, I just access its manual page using:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> tar | <span style="color: #5f8700">trim</span> 20
TAR(1)                          GNU TAR Manual                          TAR(1)
 
NAME
       tar - an archiving utility
 
SYNOPSIS
   Traditional usage
       tar {A|c|d|r|t|u|x}[GnSkUWOmpsMBiajJzZhPlRvwo] [ARG...]
 
   UNIX-style usage
       tar -A [OPTIONS] ARCHIVE ARCHIVE
 
       tar -c [-f ARCHIVE] [OPTIONS] [FILE...]
 
       tar -d [-f ARCHIVE] [OPTIONS] [FILE...]
 
       tar -t [-f ARCHIVE] [OPTIONS] [MEMBER...]
 
       tar -r [-f ARCHIVE] [OPTIONS] [FILE...]
 
… with 1147 more lines</pre>
<p>Not every command-line tool has a manual page.
Take <code>cd</code> for example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> cd
No manual entry for cd</pre>
<p>For shell builtins like <code>cd</code> you can consult the <em>zshbuiltins</em> manual page:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zshbuiltins | <span style="color: #5f8700">trim</span>
ZSHBUILTINS(1)              General Commands Manual             ZSHBUILTINS(1)
 
NAME
       zshbuiltins - zsh built-in commands
 
SHELL BUILTIN COMMANDS
       Some shell builtin commands take options as described in individual en‐
       tries; these are often referred to in the  list  below  as  `flags'  to
       avoid  confusion  with  shell options, which may also have an effect on
       the behaviour of builtin commands.  In this introductory section,  `op‐
… with 2735 more lines</pre>
<p>You can search by pressing <strong><code>/</code></strong> and exit by pressing <strong><code>q</code></strong>.
Try to find the appropriate section for <code>cd</code>.</p>
<p>Newer command-line tools often lack a manual page as well.
In that case, your best bet is to invoke the tool with the <code>--help</code> (or <code>-h</code>) option.
For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">jq</span> --help | <span style="color: #5f8700">trim</span>
jq - commandline JSON processor [version 1.6]
 
Usage:  /usr/bin/jq [options] &lt;jq filter&gt; [file...]
        /usr/bin/jq [options] --args &lt;jq filter&gt; [strings...]
        /usr/bin/jq [options] --jsonargs &lt;jq filter&gt; [JSON_TEXTS...]
 
jq is a tool for processing JSON inputs, applying the given filter to
its JSON text inputs and producing the filter's results as JSON on
standard output.
 
… with 37 more lines</pre>
<p>Specifying the <code>--help</code> option also works for command-line tools such as <code>cat</code>.
However, the corresponding man page often provides more information.
If, after trying these three approaches, you are still stuck, then it is perfectly acceptable to consult the Internet.
In the appendix, there’s a list of all command-line tools used in this book.
Besides how each command-line tool can be installed, it also shows how you can get help.</p>
<p>Manual pages can be quite verbose and difficult to read.
The tool <code>tldr</code><span class="citation"><a href="#fn32" class="footnote-ref" id="fnref32"><sup>32</sup></a></span> is a collection of community-maintained help pages for command-line tools, that aims to be a simpler, more approachable complement to traditional manual pages.
Here’s an example showing the tldr page for <code>tar</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">tldr</span> tar | <span style="color: #5f8700">trim</span> 20
 
  tar
 
  Archiving utility.
  Often combined with a compression method, such as gzip or bzip2.
  More information: https://www.gnu.org/software/tar.
 
  - [c]reate an archive and write it to a [f]ile:
    tar cf target.tar file1 file2 file3
 
  - [c]reate a g[z]ipped archive and write it to a [f]ile:
    tar czf target.tar.gz file1 file2 file3
 
  - [c]reate a g[z]ipped archive from a directory using relative paths:
    tar czf target.tar.gz --directory=path/to/directory .
 
  - E[x]tract a (compressed) archive [f]ile into the current directory [v]erbos…
    tar xvf source.tar[.gz|.bz2|.xz]
 
  - E[x]tract a (compressed) archive [f]ile into the target directory:
… with 12 more lines</pre>
<p>As you can see, rather than listing the many options alphabetically like <code>man</code> often does, <code>tldr</code> cuts to the chase by giving you a list of practical examples.</p>
</div>
</div>
<div id="summary-1" class="section level2" number="2.4">
<h2 number="2.4"><span class="header-section-number">2.4</span> Summary</h2>
<p>In this chapter you learned how to get all the required command-line tools by installing a Docker image.
I also went over some essential command-line concepts and how to get help.
Now that you have all the necessary ingredients, you’re ready for the first step of the OSEMN model for data science: obtaining data.</p>
</div>
<div id="for-further-exploration-1" class="section level2" number="2.5">
<h2 number="2.5"><span class="header-section-number">2.5</span> For Further Exploration</h2>
<ul>
<li>The subtitle of this book pays homage to the epic <em>Unix Power Tools</em> by Jerry Peek, Shelley Powers, Tim O’Reilly, and Mike Loukides. And rightly so. In 51 chapters and more than a thousand pages, it covers just about everything there is to know about Unix. It weighs over 4 pounds, so you might want to consider getting the ebook.</li>
<li>The website <a href="https://explainshell.com/">explainshell</a> parses a command or a sequence of commands and provides a short explanation of each part. Useful for quickly understanding a new command or option without having to skim through the relevant manual pages.</li>
<li>Docker is truly a brilliant piece of software. In this chapter I’ve briefly explained how to download a Docker image and run a Docker container, but it might be worthwhile to <a href="https://www.docker.com/101-tutorial">learn how to create your own Docker images</a>. The book <em>Docker: Up &amp; Running</em> by Sean Kane and Karl Matthias is a good resource as well.</li>
</ul>
<!--chapter:end:02.Rmd-->
</div>
</div>
<div id="chapter-3-obtaining-data" class="section level1" number="3">
<h1 number="3"><span class="header-section-number">3</span> Obtaining Data</h1>
<p>This chapter deals with the first step of the OSEMN model: obtaining data.
After all, without any data, there is not much data science that we can do.
I assume that the data you need to solve your data science problem already exists.
Your first task is to get this data onto your computer (and possibly also inside the Docker container) in a form that you can work with.</p>
<p>According to the Unix philosophy, text is a universal interface.
Almost every command-line tool takes text as input, produces text as output, or both.
This is the main reason why command-line tools can work so well together.
However, as we’ll see, even just text can come in multiple forms.</p>
<p>Data can be obtained in several ways—for example by downloading it from a server, querying a database, or connecting to a Web API.
Sometimes, the data comes in a compressed form or in a binary format such as a Microsoft Excel Spreadsheet.
In this chapter, I discuss several tools that help tackle this from the command line, including: <code>curl</code><span class="citation"><a href="#fn33" class="footnote-ref" id="fnref33"><sup>33</sup></a></span>, <code>in2csv</code><span class="citation"><a href="#fn34" class="footnote-ref" id="fnref34"><sup>34</sup></a></span>, <code>sql2csv</code><span class="citation"><a href="#fn35" class="footnote-ref" id="fnref35"><sup>35</sup></a></span>, and <code>tar</code><span class="citation"><a href="#fn36" class="footnote-ref" id="fnref36"><sup>36</sup></a></span>.</p>
<div id="overview" class="section level2" number="3.1">
<h2 number="3.1"><span class="header-section-number">3.1</span> Overview</h2>
<p>In this chapter, you’ll learn how to:</p>
<ul>
<li>Copy local files to the Docker image</li>
<li>Download data from the Internet</li>
<li>Decompress files</li>
<li>Extract data from spreadsheets</li>
<li>Query relational databases</li>
<li>Call web APIs</li>
</ul>
<p>This chapter starts with the following files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch03</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 924K
-rw-r--r-- 1 dst dst 627K Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">logs.tar.gz</span>
-rw-r--r-- 1 dst dst 189K Dec 14 11:46 r-datasets.db
-rw-r--r-- 1 dst dst  149 Dec 14 11:46 tmnt-basic.csv
-rw-r--r-- 1 dst dst  148 Dec 14 11:46 tmnt-missing-newline.csv
-rw-r--r-- 1 dst dst  181 Dec 14 11:46 tmnt-with-header.csv
-rw-r--r-- 1 dst dst  91K Dec 14 11:46 top2000.xlsx</pre>
<p>The instructions to get these files are in <a href="#chapter-2-getting-started">Chapter 2</a>.
Any other files are either downloaded or generated using command-line tools.</p>
</div>
<div id="copying-local-files-to-the-docker-container" class="section level2" number="3.2">
<h2 number="3.2"><span class="header-section-number">3.2</span> Copying Local Files to the Docker Container</h2>
<p>A common situation is that you already have the necessary files on your own computer.
This section explains how you can get those files into the Docker container.</p>
<p>I mentioned in <a href="#chapter-2-getting-started">Chapter 2</a> that the Docker container is an isolated virtual environment.
Luckily there is one exception to that: files can be transferred in and out the Docker container.
The local directory from which you ran <code>docker run</code>, is mapped to a directory in the Docker container.
This directory is called <em>/data</em>.
Note that this is not the home directory, which is <em>/home/dst</em>.</p>
<p>If you have one or more files on your local computer, and you want to apply some command-line tools to them, all you have to do is copy or move the files to that mapped directory.
Let’s assume that you have a file called <em>logs.csv</em> in your Downloads directory.</p>
<p>If you’re running Windows, open the command prompt or PowerShell and run the following two commands:</p>
<pre>&gt; cd %UserProfile%\Downloads
&gt; copy logs.csv MyDataScienceToolbox\</pre>
<p>If you are running Linux or macOS, open a terminal and execute the following command on your operating system (and not inside the Docker container):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> ~/Downloads/logs.csv ~/my-data-science-toolbox</pre>
<p>You can also drag-and-drop the file into the right directory using a graphical file manager such as Windows Explorer or macOS Finder.</p>
</div>
<div id="downloading-from-the-internet" class="section level2" number="3.3">
<h2 number="3.3"><span class="header-section-number">3.3</span> Downloading from the Internet</h2>
<p>The Internet provides, without a doubt, the largest resource for interesting data.
The command-line tool <code>curl</code> can be considered the command line’s Swiss Army knife when it comes to downloading data from the Internet.</p>
<div id="introducing-curl" class="section level3" number="3.3.1">
<h3 number="3.3.1"><span class="header-section-number">3.3.1</span> Introducing <code>curl</code></h3>
<p>When you browse to a URL, which stands for <em>uniform resource locator</em>, your browser interprets the data it downloads.
For example, the browser renders HTML files, plays video files automatically, and shows PDF files.
However, when you use <code>curl</code> to access a URL, it downloads the data and, by default, prints it to standard output.
<code>curl</code> doesn’t do any interpretation, but luckily other command-line tools can be used to process the data further.</p>
<p>The easiest invocation of <code>curl</code> is to specify a URL as a command-line argument.
Let’s, try downloading an article from Wikipedia:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> <span style="color: #af8700">"https://en.wikipedia.org/wiki/List_of_windmills_in_the_Netherlands"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">trim</span> <span class="callout">&#10122;</span>
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0&lt;!
DOCTYPE html&gt;
&lt;html class="client-nojs" lang="en" dir="ltr"&gt;
&lt;head&gt;
&lt;meta charset="UTF-8"/&gt;
&lt;title&gt;List of windmills in the Netherlands - Wikipedia&lt;/title&gt;
&lt;script&gt;document.documentElement.className="client-js";RLCONF={"wgBreakFrames":…
"wgRelevantPageName":"List_of_windmills_in_the_Netherlands","wgRelevantArticleI…
,"site.styles":"ready","user.styles":"ready","ext.globalCssJs.user":"ready","us…
"ext.growthExperiments.SuggestedEditSession"];&lt;/script&gt;
&lt;script&gt;(RLQ=window.RLQ||[]).push(function(){mw.loader.implement("user.options@…
100  249k    0  249k    0     0   853k      0 --:--:-- --:--:-- --:--:--  867k
… with 1754 more lines</pre>
<p><span class="callout">&#10122;</span> Remember, <code>trim</code> is used only to make the output fit nicely in the book.</p>
<p>As you can see, <code>curl</code> downloads the raw HTML returned by Wikipedia’s server; no interpretation is being done and all the contents is immediately printed on standard output.
Because of the URL, you’d think that this article would list all the windmills in the Netherlands.
However, there are apparently so many windmills left that each province has its own page.
Fascinating.</p>
<p>By default, <code>curl</code> outputs a progress meter that shows the download rate and the expected time of completion.
This output isn’t written to standard output, but a separate channel, known as <em>standard error</em>, so this doesn’t interfere when you add another tool to the pipeline.
While this information can be useful when downloading very large files, I usually find it distracting, so I specify the <code>-s</code> option to <em>silence</em> this output.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://en.wikipedia.org/wiki/List_of_windmills_in_Friesland"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">pup</span> -n <span style="color: #af8700">'table.wikitable tr'</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span></span>
234</pre>
<p><span class="callout">&#10122;</span> I’ll discuss <code>pup</code><span class="citation"><a href="#fn37" class="footnote-ref" id="fnref37"><sup>37</sup></a></span>, a handy tool for scraping websites, in more detail in <a href="#chapter-5-scrubbing-data">Chapter 5</a>.</p>
<p>And what do you know, there are apparently 234 windmills in the province of Friesland alone!</p>
</div>
<div id="saving" class="section level3" number="3.3.2">
<h3 number="3.3.2"><span class="header-section-number">3.3.2</span> Saving</h3>
<p>You can let <code>curl</code> save the output to a file by adding the <code>-O</code> option.
The filename will be based on the last part of the URL.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://en.wikipedia.org/wiki/List_of_windmills_in_Friesland"</span> -O
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 1.4M
-rw-r--r-- 1 dst dst 432K Dec 14 11:46 List_of_windmills_in_Friesland
-rw-r--r-- 1 dst dst 627K Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">logs.tar.gz</span>
-rw-r--r-- 1 dst dst 189K Dec 14 11:46 r-datasets.db
-rw-r--r-- 1 dst dst  149 Dec 14 11:46 tmnt-basic.csv
-rw-r--r-- 1 dst dst  148 Dec 14 11:46 tmnt-missing-newline.csv
-rw-r--r-- 1 dst dst  181 Dec 14 11:46 tmnt-with-header.csv
-rw-r--r-- 1 dst dst  91K Dec 14 11:46 top2000.xlsx</pre>
<p>If you don’t like that filename then you can use the <code>-o</code> option together with a filename or redirect the output to a file yourself:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://en.wikipedia.org/wiki/List_of_windmills_in_Friesland"</span> <span style="color: #af8700">&gt;</span> fries
land.html</pre>
</div>
<div id="other-protocols" class="section level3" number="3.3.3">
<h3 number="3.3.3"><span class="header-section-number">3.3.3</span> Other Protocols</h3>
<p>In total, <code>curl</code> supports <a href="https://ec.haxx.se/protocols/protocols-curl">more than 20 protocols</a>.
To download from an FTP server, which stands for File Transfer Protocol, you use <code>curl</code> the same way.
Here I download the file <em>welcome.msg</em> from <em>ftp.gnu.org</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"ftp://ftp.gnu.org/welcome.msg"</span> | <span style="color: #5f8700">trim</span>
NOTICE (Updated October 15 2021):
 
If you maintain scripts used to access ftp.gnu.org over FTP,
we strongly encourage you to change them to use HTTPS instead.
 
Eventually we hope to shut down FTP protocol access, but plan
to give notice here and other places for several months ahead
of time.
 
--
… with 19 more lines</pre>
<p>If the specified URL is a directory, <code>curl</code> will list the contents of that directory.
When the URL is password protected, you can specify a username and a password as follows with the <code>-u</code> option.</p>
<p>Or how about the DICT protocol, which allows you to access various dictionaries and lookup definitions.
Here’s the definition of “windmill” according to the Collaborative International Dictionary of English:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"dict://dict.org/d:windmill"</span> | <span style="color: #5f8700">trim</span>
220 dict.dict.org dictd 1.12.1/rf on Linux 4.19.0-10-amd64 &lt;auth.mime&gt; &lt;8291886…
250 ok
150 1 definitions retrieved
151 "Windmill" gcide "The Collaborative International Dictionary of English v.0…
Windmill \Wind"mill`\, n.
   A mill operated by the power of the wind, usually by the
   action of the wind upon oblique vanes or sails which radiate
   from a horizontal shaft. --Chaucer.
   [1913 Webster]
.
… with 2 more lines</pre>
<p>When downloading data from the Internet, however, the protocol will most likely be HTTP, so the URL will start with either <em><a href="http://" class="uri">http://</a></em> or <em><a href="https://" class="uri">https://</a></em>.</p>
</div>
<div id="following-redirects" class="section level3" number="3.3.4">
<h3 number="3.3.4"><span class="header-section-number">3.3.4</span> Following Redirects</h3>
<p>When you access a shortened URL, such as the ones that start with <em><a href="http://bit.ly/" class="uri">http://bit.ly/</a></em> or <em><a href="http://t.co/" class="uri">http://t.co/</a></em>, your browser automatically redirects you to the correct location.
With <code>curl</code>, however, you need to specify the <code>-L</code> or <code>--location</code> option in order to be redirected.
If you don’t, you can get something like:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://bit.ly/2XBxvwK"</span>
&lt;html&gt;
&lt;head&gt;&lt;title&gt;Bitly&lt;/title&gt;&lt;/head&gt;
&lt;body&gt;&lt;a href="https://youtu.be/dQw4w9WgXcQ"&gt;moved here&lt;/a&gt;&lt;/body&gt;
&lt;/html&gt;<span style="font-weight: bold; font-style: italic">%</span></pre>
<p>Sometimes you get nothing back, just like when we follow the URL mentioned above:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://youtu.be/dQw4w9WgXcQ"</span></pre>
<p>By specifying the <code>-I</code> or <code>--head</code> option, <code>curl</code> fetches only the HTTP header of the response, which allows you to inspect the status code and other information returned by the server.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sI <span style="color: #af8700">"https://youtu.be/dQw4w9WgXcQ"</span> | <span style="color: #5f8700">trim</span>
HTTP/2 303
content-type: application/binary
x-content-type-options: nosniff
cache-control: no-cache, no-store, max-age=0, must-revalidate
pragma: no-cache
expires: Mon, 01 Jan 1990 00:00:00 GMT
date: Tue, 14 Dec 2021 10:46:22 GMT
location: https://www.youtube.com/watch?v=dQw4w9WgXcQ&amp;feature=youtu.be
content-length: 0
x-frame-options: SAMEORIGIN
… with 11 more lines</pre>
<p>The first line shows the protocol followed by the HTTP status code, which is 303 in this case.
You can also see the location this URL redirects to.
Inspecting the header and getting the status code is a useful debugging tool in case <code>curl</code> does not give you the expected result.
Other common HTTP status codes include 404 (not found) and 403 (forbidden).
Wikipedia has a page that lists <a href="http://en.wikipedia.org/wiki/List_of_HTTP_status_codes">all HTTP status codes</a>.</p>
<p>In summary, <code>curl</code> is a useful command-line tool for downloading data from the Internet.
Its three most common options are <code>-s</code> to silence the progress meter, <code>-u</code> to specify a username and password, and <code>-L</code> to automatically follow redirects.
See its man page for more information (and to make your head spin):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> curl | <span style="color: #5f8700">trim</span> 20
curl(1)                           Curl Manual                          curl(1)
 
NAME
       curl - transfer a URL
 
SYNOPSIS
       curl [options / URLs]
 
DESCRIPTION
       curl  is  a tool to transfer data from or to a server, using one of the
       supported protocols (DICT, FILE, FTP, FTPS, GOPHER, HTTP, HTTPS,  IMAP,
       IMAPS,  LDAP,  LDAPS,  MQTT, POP3, POP3S, RTMP, RTMPS, RTSP, SCP, SFTP,
       SMB, SMBS, SMTP, SMTPS, TELNET and TFTP). The command  is  designed  to
       work without user interaction.
 
       curl offers a busload of useful tricks like proxy support, user authen‐
       tication, FTP upload, HTTP post, SSL connections, cookies, file  trans‐
       fer  resume,  Metalink,  and more. As you will see below, the number of
       features will make your head spin!
 
… with 3986 more lines</pre>
</div>
</div>
<div id="decompressing-files" class="section level2" number="3.4">
<h2 number="3.4"><span class="header-section-number">3.4</span> Decompressing Files</h2>
<p>If the original dataset is very large or it’s a collection of many files, the file may be a compressed archive.
Datasets which contain many repeated values (such as the words in a text file or the keys in a JSON file) are especially well suited for compression.</p>
<p>Common file extensions of compressed archives are: <em>.tar.gz</em>, <em>.zip</em>, and <em>.rar</em>.
To decompress these, you would use the command-line tools <code>tar</code>, <code>unzip</code><span class="citation"><a href="#fn38" class="footnote-ref" id="fnref38"><sup>38</sup></a></span>, and <code>unrar</code><span class="citation"><a href="#fn39" class="footnote-ref" id="fnref39"><sup>39</sup></a></span>, respectively.
(There are a few more, though less common, file extensions for which you would need yet other tools.)</p>
<p>Let’s take <em>tar.gz</em> (pronounced as “gzipped tarball”) as an example.
In order to extract an archive named <em>logs.tar.gz</em>, you would use the following incantation:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">tar</span> -xzf <span style="text-decoration: underline">logs.tar.gz</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span> </span></pre>
<p><span class="callout">&#10122;</span> It’s common to combine these three short options, like I did here, but you can also specify them separately as <code>-x -z -f</code>.
In fact, many command-tools allow you to combine options that consist of a single character.</p>
<p>Indeed, <code>tar</code> is notorious for its many command-line arguments.
In this case, the three options <code>-x</code>, <code>-z</code>, and <code>-f</code> specify that <code>tar</code> should <em>extract</em> files from an archive, use <em>gzip</em> as the decompression algorithm and use file <em>logs.tar.gz</em>.</p>
<p>However, since we’re not yet familiar with this archive, it’s a good idea to first examine its contents.
This can be done with the <code>-t</code> option (instead of the <code>-x</code> option):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">tar</span> -tzf <span style="text-decoration: underline">logs.tar.gz</span> | <span style="color: #5f8700">trim</span>
E1FOSPSAYDNUZI.2020-09-01-00.0dd00628
E1FOSPSAYDNUZI.2020-09-01-00.b717c457
E1FOSPSAYDNUZI.2020-09-01-01.05f904a4
E1FOSPSAYDNUZI.2020-09-01-02.36588daf
E1FOSPSAYDNUZI.2020-09-01-02.6cea8b1d
E1FOSPSAYDNUZI.2020-09-01-02.be4bc86d
E1FOSPSAYDNUZI.2020-09-01-03.16f3fa32
E1FOSPSAYDNUZI.2020-09-01-03.1c0a370f
E1FOSPSAYDNUZI.2020-09-01-03.76df64bf
E1FOSPSAYDNUZI.2020-09-01-04.0a1ade1b
… with 2427 more lines</pre>
<p>Is seems that this archive contains a lot of files, and they are not inside a directory.
In order to keep the current directory clean, it’s a good idea to first create a new directory using <code>mkdir</code> and extract those files there using the <code>-C</code> option.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">mkdir</span> logs
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">tar</span> -xzf <span style="text-decoration: underline">logs.tar.gz</span> -C <span style="text-decoration: underline">logs</span></pre>
<p>Let’s verify the number of files and some of their contents:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">ls</span> <span style="text-decoration: underline">logs</span> | <span style="color: #5f8700">wc</span> -l
2437
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> logs/<span style="color: #0087ff">*</span> | <span style="color: #5f8700">trim</span>
#Version: 1.0
#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem…
2020-09-01      00:51:54        SEA19-C1        391     206.55.174.150  GET    …
2020-09-01      00:54:59        CPH50-C2        384     82.211.213.95   GET    …
#Version: 1.0
#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem…
2020-09-01      00:04:28        DFW50-C1        391     2a03:2880:11ff:9::face:…
#Version: 1.0
#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem…
2020-09-01      01:04:14        ATL56-C4        385     2600:1700:2760:da20:548…
… with 10279 more lines</pre>
<p>Excellent.
Now, I understand that you’d like to scrub and explore these log files, but that’s for later in <a href="#chapter-5-scrubbing-data">Chapter 5</a> and <a href="#chapter-7-exploring-data">Chapter 7</a>.</p>
<p>In time, you’ll get used to these options, but I’d like to show you an alternative option, which might be convenient.
Rather than remembering the different command-line tools and their options, there’s a handy script called <code>unpack</code><span class="citation"><a href="#fn40" class="footnote-ref" id="fnref40"><sup>40</sup></a></span>, which will decompress many different formats.
<code>unpack</code> looks at the extension of the file that you want to decompress, and calls the appropriate command-line tool.
Now, in order to decompress this same file, you would run:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">unpack</span> <span style="text-decoration: underline">logs.tar.gz</span></pre>
</div>
<div id="converting-microsoft-excel-spreadsheets-to-csv" class="section level2" number="3.5">
<h2 number="3.5"><span class="header-section-number">3.5</span> Converting Microsoft Excel Spreadsheets to CSV</h2>
<p>For many people, Microsoft Excel offers an intuitive way to work with small datasets and perform calculations on them.
As a result, a lot of data is embedded into Microsoft Excel spreadsheets.
These spreadsheets are, depending on the extension of the filename, stored in either a proprietary binary format (<em>.xls</em>) or as a collection of compressed XML files (<em>.xlsx</em>).
In both cases, the data is not readily usable by most command-line tools.
It would be a shame if we could not use those valuable datasets just because they are stored this way.</p>
<p>Especially when you’re just starting out at the command line, you might be tempted to convert your spreadsheet to CSV by opening it in Microsoft Excel or an open source variant such as LibreOffice Calc, and manually exporting it to CSV.
While this works as a one-off solution, the disadvantage is that it does not scale well to multiple files and cannot be automated.
Furthermore, when you’re working on a server, chances are that you don’t have such an application available.
Trust me, you’ll get the hang of it.</p>
<p>Luckily, there is a command-line tool called <code>in2csv</code>, that converts Microsoft Excel spreadsheets to CSV files.
CSV stands for comma-separated values.
Working with CSV can be tricky because it lacks a formal specification.
<!-- TODO: MUST: check and possibly fix reference? -->
Yakov Shafranovich defines the CSV format according to the following three points:<span class="citation"><a href="#fn41" class="footnote-ref" id="fnref41"><sup>41</sup></a></span></p>
<ol style="list-style-type: decimal">
<li>Each record is located on a separate line, delimited by a line break (<code>␊</code>). Take, for example, the following CSV file with crucial information about the Teenage Mutant Ninja Turtles:</li>
</ol>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> -A <span style="text-decoration: underline">tmnt-basic.csv</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span></span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">tmnt-basic.csv</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ Leonardo,Leo,blue,two<span style="color: #00afaf">·</span>ninjakens<span style="color: #af005f">␊</span>
   2   │ Raphael,Raph,red,pair<span style="color: #00afaf">·</span>of<span style="color: #00afaf">·</span>sai<span style="color: #af005f">␊</span>
   3   │ Michelangelo,Mikey<span style="color: #00afaf">·</span>or<span style="color: #00afaf">·</span>Mike,orange,pair<span style="color: #00afaf">·</span>of<span style="color: #00afaf">·</span>nunchaku<span style="color: #af005f">␊</span>
   4   │ Donatello,Donnie<span style="color: #00afaf">·</span>or<span style="color: #00afaf">·</span>Don,purple,staff<span style="color: #af005f">␊</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p><span class="callout">&#10122;</span> The <code>-A</code> option makes <code>bat</code> show all non-printable characters like spaces, tabs, and newlines.</p>
<ol start="2" style="list-style-type: decimal">
<li>The last record in the file may or may not have an ending line break. For example:</li>
</ol>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> -A <span style="text-decoration: underline">tmnt-missing-newline.csv</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">tmnt-missing-newline.csv</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ Leonardo,Leo,blue,two<span style="color: #00afaf">·</span>ninjakens<span style="color: #af005f">␊</span>
   2   │ Raphael,Raph,red,pair<span style="color: #00afaf">·</span>of<span style="color: #00afaf">·</span>sai<span style="color: #af005f">␊</span>
   3   │ Michelangelo,Mikey<span style="color: #00afaf">·</span>or<span style="color: #00afaf">·</span>Mike,orange,pair<span style="color: #00afaf">·</span>of<span style="color: #00afaf">·</span>nunchaku<span style="color: #af005f">␊</span>
   4   │ Donatello,Donnie<span style="color: #00afaf">·</span>or<span style="color: #00afaf">·</span>Don,purple,staff
───────┴────────────────────────────────────────────────────────────────────────</pre>
<ol start="3" style="list-style-type: decimal">
<li>There may be a header appearing as the first line of the file with the same format as normal record lines. This header will contain names corresponding to the fields in the file and should contain the same number of fields as the records in the rest of the file. For example:</li>
</ol>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> -A <span style="text-decoration: underline">tmnt-with-header.csv</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">tmnt-with-header.csv</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ name,nickname,mask_color,weapon<span style="color: #af005f">␊</span>
   2   │ Leonardo,Leo,blue,two<span style="color: #00afaf">·</span>ninjakens<span style="color: #af005f">␊</span>
   3   │ Raphael,Raph,red,pair<span style="color: #00afaf">·</span>of<span style="color: #00afaf">·</span>sai<span style="color: #af005f">␊</span>
   4   │ Michelangelo,Mikey<span style="color: #00afaf">·</span>or<span style="color: #00afaf">·</span>Mike,orange,pair<span style="color: #00afaf">·</span>of<span style="color: #00afaf">·</span>nunchaku<span style="color: #af005f">␊</span>
   5   │ Donatello,Donnie<span style="color: #00afaf">·</span>or<span style="color: #00afaf">·</span>Don,purple,staff<span style="color: #af005f">␊</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>As you can see, CSV by default is not too readable.
You can pipe the data to a tool called <code>csvlook</code><span class="citation"><a href="#fn42" class="footnote-ref" id="fnref42"><sup>42</sup></a></span>, which will nicely format it into a table.
If the CSV data has no header, like <em>tmnt-missing-newline.csv</em> then you need to add the <code>-H</code> option, otherwise the first line will be interpreted as the header.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> <span style="text-decoration: underline">tmnt-with-header.csv</span>
│ name         │ nickname      │ mask_color │ weapon           │
├──────────────┼───────────────┼────────────┼──────────────────┤
│ Leonardo     │ Leo           │ blue       │ two ninjakens    │
│ Raphael      │ Raph          │ red        │ pair of sai      │
│ Michelangelo │ Mikey or Mike │ orange     │ pair of nunchaku │
│ Donatello    │ Donnie or Don │ purple     │ staff            │
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> <span style="text-decoration: underline">tmnt-basic.csv</span>
│ Leonardo     │ Leo           │ blue   │ two ninjakens    │
├──────────────┼───────────────┼────────┼──────────────────┤
│ Raphael      │ Raph          │ red    │ pair of sai      │
│ Michelangelo │ Mikey or Mike │ orange │ pair of nunchaku │
│ Donatello    │ Donnie or Don │ purple │ staff            │
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> -H <span style="text-decoration: underline">tmnt-missing-newline.csv</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span></span>
│ a            │ b             │ c      │ d                │
├──────────────┼───────────────┼────────┼──────────────────┤
│ Leonardo     │ Leo           │ blue   │ two ninjakens    │
│ Raphael      │ Raph          │ red    │ pair of sai      │
│ Michelangelo │ Mikey or Mike │ orange │ pair of nunchaku │
│ Donatello    │ Donnie or Don │ purple │ staff            │</pre>
<p><span class="callout">&#10122;</span> The <code>-H</code> option specifies that the CSV file has no header.</p>
<p>Let’s demonstrate <code>in2csv</code> using a spreadsheet that contains the 2000 most popular songs according to an annual Dutch marathon radio program <a href="https://www.top2000nl.com">Top 2000</a>.
To extract its data, you invoke <code>in2csv</code> as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> <span style="color: #af8700">"https://www.nporadio2.nl/data/download/TOP-2000-2020.xlsx"</span> <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">top2000.xls
x</span>
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 41744  100 41744    0     0  46024      0 --:--:-- --:--:-- --:--:-- 46228
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">in2csv</span> <span style="text-decoration: underline">top2000.xlsx</span> | <span style="color: #5f8700">tee</span> top2000.csv | <span style="color: #5f8700">trim</span>
BadZipFile: File is not a zip file</pre>
<p>Who is Danny Vera? The most popular song is supposed to be <em>Bohemian Rhapsody</em>, of course.
Well, at least Queen appears plenty of times in the Top 2000 so I can’t really complain:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvgrep</span> <span style="text-decoration: underline">top2000.csv</span> --columns ARTIEST --regex <span style="color: #af8700">'^Queen$'</span> | <span style="color: #5f8700">csvlook</span> -I <span class="callout">&#10122;</span>
StopIteration:
StopIteration:</pre>
<p><span class="callout">&#10122;</span> The value after the <code>--regex</code> options is a regular expression (or <em>regex</em>). It’s a special syntax for defining patterns. Here, I only want to match artists that exactly match “Queen,” so I use the caret (<code>^</code>) and dollar sign (<code>$</code>) to match the start and end of the values in the column <em><code>ARTIEST</code></em>.</p>
<p>By the way, the tools <code>in2csv</code>, <code>csvgrep</code>, and <code>csvlook</code> are part of CSVkit, which is a collection of command-line tools to work with CSV data.</p>
<p>The format of the file is automatically determined by the extension, <em>.xlsx</em> in this case.
If you were to pipe the data into <code>in2csv</code>, you would have to specify the format explicitly.</p>
<p>A spreadsheet can contain multiple worksheets.
<code>in2csv</code> extracts, by default, the first worksheet.
To extract a different worksheet, you need to pass the name of worksheet to the <code>--sheet</code> option.
If you’re not sure what’s the worksheet is called, you can use the <code>--names</code> option, which prints the names of all the worksheets.
Here we see that <em>top2000.xlsx</em> has only one sheet, named <em>Blad1</em> (which is Dutch for <em>Sheet1</em>).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">in2csv</span> --names <span style="text-decoration: underline">top2000.xlsx</span>
BadZipFile: File is not a zip file
sys:1: ResourceWarning: unclosed file &lt;_io.BufferedReader name='top2000.xlsx'&gt;</pre>
</div>
<div id="querying-relational-databases" class="section level2" number="3.6">
<h2 number="3.6"><span class="header-section-number">3.6</span> Querying Relational Databases</h2>
<p>Many companies store their data in a relational database.
Just as with spreadsheets, it would be great if we could obtain that data from the command line.</p>
<p>Examples of relational databases are MySQL, PostgreSQL, and SQLite.
These databases all have a slightly different way of interfacing with them.
Some provide a command-line tool or a command-line interface, while others do not.
Moreover, they are not very consistent when it comes to their usage and output.</p>
<p>Fortunately, there is a command-line tool called <code>sql2csv</code>, which is part of the CSVkit suite.
It works with many different databases through a common interface, including MySQL, Oracle, PostgreSQL, SQLite, Microsoft SQL Server, and Sybase.
The output of <code>sql2csv</code> is, as its name suggests, in CSV format.</p>
<p>We can obtain data from relational databases by executing a <em><code>SELECT</code></em> query on them.
(<code>sql2csv</code> also support <em><code>INSERT</code></em>, <em><code>UPDATE</code></em>, and <em><code>DELETE</code></em> queries, but that’s not the purpose of this chapter.)</p>
<p><code>sql2csv</code> needs two arguments: <code>--db</code>, which specifies the database URL, of which the typical form is: <code>dialect+driver://username:password@host:port/database</code> and <code>--query</code>, which contains the <em><code>SELECT</code></em> query.
For example, given an SQLite database that contains the standard datasets from R<a href="#fn43" class="footnote-ref" id="fnref43"><sup>43</sup></a> , I can select all the rows from the table <em>mtcars</em> and sort them by the <em>mpg</em> column as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">sql2csv</span> --db <span style="color: #af8700">'sqlite:///r-datasets.db'</span> \
<span style="font-weight: bold">&gt;</span> --query <span style="color: #af8700">'SELECT row_names AS car, mpg FROM mtcars ORDER BY mpg'</span> | <span style="color: #5f8700">csvlook</span>
│ car                 │  mpg │
├─────────────────────┼──────┤
│ Cadillac Fleetwood  │ 10.4 │
│ Lincoln Continental │ 10.4 │
│ Camaro Z28          │ 13.3 │
│ Duster 360          │ 14.3 │
│ Chrysler Imperial   │ 14.7 │
│ Maserati Bora       │ 15.0 │
│ Merc 450SLC         │ 15.2 │
│ AMC Javelin         │ 15.2 │
… with 24 more lines</pre>
<p>This SQLite database is a local file, so in this I don’t need to specify any username, password or host.
If you wish to query the database of your employer, you’d of course need know how you can access it and you’d need the permission to do so.</p>
</div>
<div id="calling-web-apis" class="section level2" number="3.7">
<h2 number="3.7"><span class="header-section-number">3.7</span> Calling Web APIs</h2>
<p>In the previous section I explained how to download files from the Internet.
Another way data can come from the Internet is through a web API, which stands for <em>Application Programming Interface</em>.
The number of APIs that are being offered is growing at an increasing rate, which means a lot of interesting data for us data scientists.</p>
<p>Web APIs are not meant to be presented in a nice layout, such as websites.
Instead, most web APIs return data in a structured format, such as JSON or XML.
Having data in a structured form has the advantage that the data can be easily processed by other tools, such as <code>jq</code>.
For example, the API of Ice and Fire, which contains a lot of information about George R.R. Martin’s fictional world, in which the Game of Thrones book and TV shows take place, returns data in the following JSON structure:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://anapioficeandfire.com/api/characters/583"</span> | <span style="color: #5f8700">jq</span> <span style="color: #af8700">'.'</span>
<span style="font-weight: bold">{</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"url"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">"https://anapioficeandfire.com/api/characters/583"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"name"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Jon Snow"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"gender"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Male"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"culture"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Northmen"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"born"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">"In 283 AC"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"died"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">""</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span> <span class="callout">&#10122;</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"titles"</span><span style="font-weight: bold">: [</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Lord Commander of the Night's Watch"</span>
<span style="font-weight: bold">  ],</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"aliases"</span><span style="font-weight: bold">: [</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Lord Snow"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Ned Stark's Bastard"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"The Snow of Winterfell"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"The Crow-Come-Over"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"The 998th Lord Commander of the Night's Watch"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"The Bastard of Winterfell"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"The Black Bastard of the Wall"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Lord Crow"</span>
<span style="font-weight: bold">  ],</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"father"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">""</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"mother"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">""</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"spouse"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">""</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"allegiances"</span><span style="font-weight: bold">: [</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"https://anapioficeandfire.com/api/houses/362"</span>
<span style="font-weight: bold">  ],</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"books"</span><span style="font-weight: bold">: [</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"https://anapioficeandfire.com/api/books/5"</span>
<span style="font-weight: bold">  ],</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"povBooks"</span><span style="font-weight: bold">: [</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"https://anapioficeandfire.com/api/books/1"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"https://anapioficeandfire.com/api/books/2"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"https://anapioficeandfire.com/api/books/3"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"https://anapioficeandfire.com/api/books/8"</span>
<span style="font-weight: bold">  ],</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"tvSeries"</span><span style="font-weight: bold">: [</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Season 1"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Season 2"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Season 3"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Season 4"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Season 5"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Season 6"</span>
<span style="font-weight: bold">  ],</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"playedBy"</span><span style="font-weight: bold">: [</span>
<span style="font-weight: bold">    </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Kit Harington"</span>
<span style="font-weight: bold">  ]</span>
<span style="font-weight: bold">}</span></pre>
<p><span class="callout">&#10122;</span> Spoiler alert: this data is not entirely up-to-date.</p>
<p>The data is piped to a command-line tool <code>jq</code> just to display it in a nice way.
<code>jq</code> has many more scrubbing and exploring possibilities that I will explore in <a href="#chapter-5-scrubbing-data">Chapter 5</a> and <a href="#chapter-7-exploring-data">Chapter 7</a>.</p>
<div id="authentication" class="section level3" number="3.7.1">
<h3 number="3.7.1"><span class="header-section-number">3.7.1</span> Authentication</h3>
<p>Some web APIs require you to authenticate (that is, to prove your identity) before you can consume their output.
There are several ways to do this.
Some web APIs use API keys while others use the OAuth protocol.
News API, an independent source of headlines and news articles, is a great example.
Let’s see what happens when you try to access this API without an API key:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"http://newsapi.org/v2/everything?q=linux"</span> | <span style="color: #5f8700">jq</span> <span style="text-decoration: underline">.</span>
<span style="font-weight: bold">{</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"status"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">"error"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,
  </span><span style="font-weight: bold; color: #0087ff">"code"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">"apiKeyMissing"</span><span style="font-weight: bold; color: #5f8700"></span><span style="font-weight: bold">,</span>
<span style="font-weight: bold">  </span><span style="font-weight: bold; color: #0087ff">"message"</span><span style="font-weight: bold">: </span><span style="color: #5f8700"></span><span style="color: #5f8700">"Your API key is missing. Append this to the URL with the apiKey pa
ram, or use the x-api-key HTTP header."</span>
<span style="font-weight: bold">}</span></pre>
<p>Well, that was to be expected.
The part after the question mark, by the way, is where we pass any query parameters.
That’s also the place where you need to specify an API key.
I’d like to keep my own API key a secret, so I insert it below by reading the file <em>/data/.secret/newsapi.org_apikey</em> using command substitution.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"http://newsapi.org/v2/everything?q=linux&amp;apiKey=</span><span style="color: #af005f">$(</span><span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">/data/.secret/new
sapi.org_apikey</span><span style="color: #af005f"></span><span style="color: #af005f">)</span><span style="color: #af8700">"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">jq</span> <span style="color: #af8700">'.'</span> | <span style="color: #5f8700">trim</span> 30
{
  "status": "ok",
  "totalResults": 9653,
  "articles": [
    {
      "source": {
        "id": "engadget",
        "name": "Engadget"
      },
      "author": "Igor Bonifacic",
      "title": "'Arma 3' and 'DayZ' add BattlEye anti-cheat support through Val…
      "description": "While there are still many unknowns about Steam Deck\r\n
      "url": "https://www.engadget.com/arma-3-dayz-proton-battleye-support-2246…
      "urlToImage": "https://s.yimg.com/os/creatr-uploaded-images/2021-12/bcb0f…
      "publishedAt": "2021-12-03T22:46:25Z",
      "content": "While there are still many unknowns about Steam Deck\r\ns lib…
    },
    {
      "source": {
        "id": null,
        "name": "Slashdot.org"
      },
      "author": "EditorDavid",
      "title": "AWS Embraces Fedora Linux for Its Cloud-Based 'Amazon Linux'",
      "description": "ZDNet reports:\n\nBy and large, the public cloud runs on …
      "url": "https://linux.slashdot.org/story/21/11/27/0328223/aws-embraces-fe…
      "urlToImage": "https://a.fsdn.com/sd/topics/cloud_64.png",
      "publishedAt": "2021-11-27T16:34:00Z",
      "content": "By and large, the public cloud runs on Linux. Most users, eve…
    },
… with 236 more lines</pre>
<p>You can get your own API key at <a href="https://newsapi.org">News API’s website</a>.</p>
</div>
<div id="streaming-apis" class="section level3" number="3.7.2">
<h3 number="3.7.2"><span class="header-section-number">3.7.2</span> Streaming APIs</h3>
<p>Some web APIs return data in a streaming manner.
This means that once you connect to it, the data will continue to pour in, until the connection is closed.
A well-known example is the Twitter “firehose,” which constantly streams all the tweets being sent around the world.
Luckily, most command-line tools also operate in a streaming matter.</p>
<p>Let’s take a 10 second sample of one of Wikimedia’s streaming APIs, for example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://stream.wikimedia.org/v2/stream/recentchange"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sample</span> -s 10 <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">wikimedia-stream-sample</span></pre>
<p>This particular API returns all changes that have been made to Wikipedia and other properties of Wikimedia.
The command-line tool <code>sample</code> is used to close the connection after 10 seconds.
The connection can also be closed manually by pressing <strong><code>Ctrl-C</code></strong> to send an interrupt.
The output is saved to a file <em>wikimedia-stream-sample</em>.
Let’s take a peek using <code>trim</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wikimedia-stream-sample</span> <span style="color: #5f8700">trim</span>
:ok
 
event: message
id: [{"topic":"eqiad.mediawiki.recentchange","partition":0,"timestamp":16101133…
data: {"$schema":"/mediawiki/recentchange/1.0.0","meta":{"uri":"https://en.wiki…
 
event: message
id: [{"topic":"eqiad.mediawiki.recentchange","partition":0,"timestamp":16101133…
data: {"$schema":"/mediawiki/recentchange/1.0.0","meta":{"uri":"https://www.wik…
 
… with 1078 more lines</pre>
<p>With a little bit of <code>sed</code> and <code>jq</code> I can scrub this data to get a glimpse of the changes happening on the English version of Wikipedia:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wikimedia-stream-sample</span> <span style="color: #5f8700">sed</span> -n <span style="color: #af8700">'s/^data: //p'</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">jq</span> <span style="color: #af8700">'select(.type == "edit" and .server_name == "en.wikipedia.org") | .title'</span> <span class="callout">&#10123;</span>
<span style="color: #5f8700">"Odion Ighalo"</span>
<span style="color: #5f8700">"Hold Up (song)"</span>
<span style="color: #5f8700">"Talk:Royal Bermuda Yacht Club"</span>
<span style="color: #5f8700">"Jenna Ushkowitz"</span>
<span style="color: #5f8700">"List of films released by Yash Raj Films"</span>
<span style="color: #5f8700">"SP.A"</span>
<span style="color: #5f8700">"Odette (musician)"</span>
<span style="color: #5f8700">"Talk:Pierre Avoi"</span>
<span style="color: #5f8700">"User:Curlymanjaro/sandbox3"</span>
<span style="color: #5f8700">"List of countries by electrification rate"</span>
<span style="color: #5f8700">"Grieg (crater)"</span>
<span style="color: #5f8700">"Gorman, Edmonton"</span>
<span style="color: #5f8700">"Khabza"</span>
<span style="color: #5f8700">"QAnon"</span>
<span style="color: #5f8700">"Khaw Boon Wan"</span>
<span style="color: #5f8700">"Draft:Oggy and the Cockroaches (1975 TV series)"</span>
<span style="color: #5f8700">"Renzo Reggiardo"</span>
<span style="color: #5f8700">"Greer, Arizona"</span>
<span style="color: #5f8700">"National Curriculum for England"</span>
<span style="color: #5f8700">"Mod DB"</span>
<span style="color: #5f8700">"Jordanian Pro League"</span>
<span style="color: #5f8700">"List of foreign Serie A players"</span></pre>
<p><span class="callout">&#10122;</span> This <code>sed</code> expression only prints lines that start with <em><code>data:</code></em> and prints the part after the semicolon, which happens to be JSON.
<br><span class="callout">&#10123;</span> This <code>jq</code> expression prints the <em><code>title</code></em> key of JSON objects that have a certain <em><code>type</code></em> and <em><code>server_name</code></em>.</p>
<p>Speaking of streaming, did you know that you could stream <em>Star Wars: Episode IV – A New Hope</em> for free using <code>telnet</code><span class="citation"><a href="#fn44" class="footnote-ref" id="fnref44"><sup>44</sup></a></span>?</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">telnet</span> <span style="text-decoration: underline">towel.blinkenlights.nl</span></pre>
<p>And after some time, we see that Han Solo did shoot first!</p>
<pre> 
 
                       -===                    `"',
       I'll bet you   ""o o                    O O|)
           have!      _\ -/_                  _\o/ _
                     || || |*                /|\ / |\
                     \\ || ***              //| |  |\\
                      \\o=***********      // | |  | ||
                      |\(#'***\\        -==#  | |  | ||
                      |====|*  ')         '\  |====| /#
                      |/|| |                  | || |  "
                      ( )( )                  | || |
                      |-||-|                  | || |
                      | || |                  | || |
      ________________[_][__\________________/__)(_)_____________________
 </pre>
<p>Sure, it’s probably not a good source of data, but there’s nothing wrong with enjoying an old classic while training your machine learning models<a href="#fn45" class="footnote-ref" id="fnref45"><sup>45</sup></a>.</p>
</div>
</div>
<div id="summary-2" class="section level2" number="3.8">
<h2 number="3.8"><span class="header-section-number">3.8</span> Summary</h2>
<p>Congratulations, you have finished the first step of the OSEMN model.
You’ve learned a variety of ways to obtain data, ranging from downloading to querying a relational database.
In the next chapter, which is an intermezzo chapter, I’ll teach you how to create your own command-line tools.
Feel free to skip this and go to <a href="#chapter-5-scrubbing-data">Chapter 5</a> (the second step of the OSEMN model) if you cannot wait to learn about scrubbing data.</p>
</div>
<div id="for-further-exploration-2" class="section level2" number="3.9">
<h2 number="3.9"><span class="header-section-number">3.9</span> For Further Exploration</h2>
<ul>
<li>Looking for a dataset to practice on? The GitHub repository <a href="https://github.com/awesomedata/awesome-public-datasets"><em>Awesome Public Datasets</em></a> lists hundreds of high-quality datasets that are publicly available.</li>
<li>Or perhaps you’d rather practice with an API? The GitHub repository <a href="https://github.com/public-apis/public-apis"><em>Public APIs</em></a> lists many free APIs. <a href="http://api.citybik.es/v2/">City Bikes</a> and <a href="https://the-one-api.dev/">The One API</a> are among my favorites.</li>
<li>Writing SQL queries to obtain data from a relational database is an important skill. The first 15 lessons of the book <em>SQL in 10 Minutes a Day</em> by Ben Forta teach the <em><code>SELECT</code></em> statement and its filtering, grouping, and sorting capabilities.</li>
</ul>
<!--chapter:end:03.Rmd-->
<!-- TODO: Remove dashes from filenames top-words.sh etc. -->
</div>
</div>
<div id="chapter-4-creating-command-line-tools" class="section level1" number="4">
<h1 number="4"><span class="header-section-number">4</span> Creating Command-line Tools</h1>
<p>Throughout the book, I’ll introduce you to many commands and pipelines that basically fit on one line.
These are known as one-liners or pipelines.
Being able to perform complex tasks with just a one-liner is what makes the command line powerful.
It’s a very different experience from writing and using traditional programs.</p>
<p>Some tasks you perform only once, and some you perform more often.
Some tasks are very specific and others can be generalized.
If you need to repeat a certain one-liner on a regular basis, it’s worthwhile to turn this into a command-line tool of its own.
So, both one-liners and command-line tools have their uses.
Recognizing the opportunity requires practice and skill.
The advantages of a command-line tool are that you don’t have to remember the entire one-liner and that it improves readability if you include it into some other pipeline.
In that sense, you can think of a command-line tool as similar to a function in a programming language.</p>
<p>The benefit of a working with a programming language, however, is that the code is in one or more file.
This means that you can easily edit and reuse that code.
If the code has parameters it can even be generalized and re-applied to problems that follow a similar pattern.</p>
<p>Command-line tools have the best of both worlds: they can be used from the command line, accept parameters, and only have to be created once.
In this chapter, you’re going to get familiar with creating command-line tools in two ways.
First, I explain how to turn those one-liners into reusable command-line tools.
By adding parameters to your commands, you can add the same flexibility that a programming language offers.
Subsequently, I demonstrate how to create reusable command-line tools from code that’s written in a programming language.
By following the Unix philosophy, your code can be combined with other command-line tools, which may be written in an entirely different language.
In this chapter, I will focus on three programming languages: Bash, Python, and R.</p>
<p>I believe that creating reusable command-line tools makes you a more efficient and productive data scientist in the long run.
You will gradually build up your own data science toolbox from which you can draw existing tools and apply it to problems you have encountered previously.
It requires practice to recognize the opportunity to turn a one-liner or existing code into a command-line tool.</p>

<div class="rmdtip">
In order to turn a one-liner into a shell script, I’m going to use a tiny bit of shell scripting.
This book only demonstrates a small subset of concepts from shell scripting, including variables, conditionals, and loops.
A complete course in shell scripting deserves a book on its own, and is therefore beyond the scope of this one.
If you want to dive deeper into shell scripting, I recommend the book <em>Classic Shell Scripting</em> by <span class="citation"><a href="#ref-Robbins2005" role="doc-biblioref">Arnold Robbins and Nelson H. F. Beebe</a><a href="#fn46" class="footnote-ref" id="fnref46"><sup>46</sup></a></span>.
</div>
<div id="overview-1" class="section level2" number="4.1">
<h2 number="4.1"><span class="header-section-number">4.1</span> Overview</h2>
<p>In this chapter, you’ll learn how to:</p>
<ul>
<li>Convert one-liners into parameterized shell scripts</li>
<li>Turn existing Python and R code into reusable command-line tools</li>
</ul>
<p>This chapter starts with the following files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch04</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 32K
-rwxr-xr-x 1 dst dst 400 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">fizzbuzz.py</span>*
-rwxr-xr-x 1 dst dst 391 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">fizzbuzz.R</span>*
-rwxr-xr-x 1 dst dst 182 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">stream.py</span>*
-rwxr-xr-x 1 dst dst 147 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">stream.R</span>*
-rwxr-xr-x 1 dst dst 105 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words-4.sh</span>*
-rwxr-xr-x 1 dst dst 128 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words-5.sh</span>*
-rwxr-xr-x 1 dst dst 647 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words.py</span>*
-rwxr-xr-x 1 dst dst 584 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words.R</span>*</pre>
<p>The instructions to get these files are in <a href="#chapter-2-getting-started">Chapter 2</a>.
Any other files are either downloaded or generated using command-line tools.</p>
</div>
<div id="converting-one-liners-into-shell-scripts" class="section level2" number="4.2">
<h2 number="4.2"><span class="header-section-number">4.2</span> Converting One-liners into Shell Scripts</h2>
<p>In this section I’m going to explain how to turn a one-liner into a reusable command-line tool.
Let’s say that you would like to get the top most frequent words used in a piece of text.
Take the book <em>Alice’s Adventures in Wonderland</em> by Lewis Carroll, which is, like many other great books, freely available on Project Gutenberg.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> | <span style="color: #5f8700">trim</span>
﻿The Project Gutenberg eBook of Alice’s Adventures in Wonderland, by Lewis …
 
This eBook is for the use of anyone anywhere in the United States and
most other parts of the world at no cost and with almost no restrictions
whatsoever. You may copy it, give it away or re-use it under the terms
of the Project Gutenberg License included with this eBook or online at
www.gutenberg.org. If you are not located in the United States, you
will have to check the laws of the country where you are located before
using this eBook.
 
… with 3751 more lines</pre>
<p>The following sequence of tools or <em>pipeline</em> should do the job:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> | <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[:upper:]'</span> <span style="color: #af8700">'[:lower:]'</span> | <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10123;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">grep</span> -oE <span style="color: #af8700">"[a-z\']{2,}"</span> | <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10124;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> | <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10125;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">uniq</span> -c | <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10126;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> -nr | <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10127;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">head</span> -n 10 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10128;</span></span>
   1839 the
    942 and
    811 to
    638 of
    610 it
    553 she
    486 you
    462 said
    435 in
    403 alice</pre>
<p><span class="callout">&#10122;</span> Downloading an ebook using <code>curl</code>.
<br><span class="callout">&#10123;</span> Converting the entire text to lowercase using <code>tr</code><span class="citation"><a href="#fn47" class="footnote-ref" id="fnref47"><sup>47</sup></a></span>.
<br><span class="callout">&#10124;</span> Extracting all the words using <code>grep</code><span class="citation"><a href="#fn48" class="footnote-ref" id="fnref48"><sup>48</sup></a></span> and put each word on separate line.
<br><span class="callout">&#10125;</span> Sort these words in alphabetical order using <code>sort</code><span class="citation"><a href="#fn49" class="footnote-ref" id="fnref49"><sup>49</sup></a></span>.
<br><span class="callout">&#10126;</span> Remove all the duplicates and count how often each word appears in the list using <code>uniq</code><span class="citation"><a href="#fn50" class="footnote-ref" id="fnref50"><sup>50</sup></a></span>.
<br><span class="callout">&#10127;</span> Sort this list of unique words by their count in descending order using <code>sort</code>.
<br><span class="callout">&#10128;</span> Keep only the top 10 lines (i.e., words) using <code>head</code>.</p>
<p>Those words indeed appear the most often in the text.
Because those words (apart from the word “alice”) appear very frequently in many English texts, they carry very little meaning.
In fact, these are known as <em>stopwords</em>.
If we get rid of those, we keep the most frequent words that are related to this text.</p>
<p>Here’s a list of stopwords I’ve found:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">"https://raw.githubusercontent.com/stopwords-iso/stopwords-en/master/
stopwords-en.txt"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> | <span style="color: #5f8700">tee</span> stopwords | <span style="color: #5f8700">trim</span> 20
10
39
a
able
ableabout
about
above
abroad
abst
accordance
according
accordingly
across
act
actually
ad
added
adj
adopted
ae
… with 1278 more lines</pre>
<p>With <code>grep</code> we can filter out the stopwords right before we start counting:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[:upper:]'</span> <span style="color: #af8700">'[:lower:]'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">grep</span> -oE <span style="color: #af8700">"[a-z\']{2,}"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">grep</span> -Fvwf <span style="text-decoration: underline">stopwords</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">uniq</span> -c |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> -nr |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">head</span> -n 10
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
     63 king
     60 turtle
     57 mock
     56 hatter
     55 gryphon</pre>
<p><span class="callout">&#10122;</span> Obtain the patterns from a file (<em>stopwords</em> in our case), one per line, with <code>-f</code>. Interpret those patterns as fixed strings with <code>-F</code>. Select only those lines containing matches that form whole words with <code>-w</code>. Select non-matching lines with <code>-v</code>.</p>

<div class="rmdtip">
Each command-line tool used in this one-liner offers a man page.
So in case you would like to know more about, say, <code>grep</code>, you can run <code>man grep</code> from the command line.
The command-line tools <code>tr</code>, <code>grep</code>, <code>uniq</code>, and <code>sort</code> will be discussed in more detail in the next chapter.
</div>
<p>There is nothing wrong with running this one-liner just once.
However, imagine if you wanted to have the top 10 words of every e-book on Project Gutenberg.
Or imagine that you wanted the top 10 words of a news website on a hourly basis.
In those cases, it would be best to have this one-liner as a separate building block that can be part of something bigger.
To add some flexibility to this one-liner in terms of parameters, let’s turn it into a shell script.</p>
<p>This allows us to take the one-liner as the starting point, and gradually improve on it.
To turn this one-liner into a reusable command-line tool, I’ll walk you through the following six steps:</p>
<ol style="list-style-type: decimal">
<li>Copy and paste the one-liner into a file.</li>
<li>Add execute permissions.</li>
<li>Define a so-called shebang.</li>
<li>Remove the fixed input part.</li>
<li>Add a parameter.</li>
<li>Optionally extend your PATH.</li>
</ol>
<div id="step-1-create-file" class="section level3" number="4.2.1">
<h3 number="4.2.1"><span class="header-section-number">4.2.1</span> Step 1: Create File</h3>
<p>The first step is to create a new file.
You can open your favorite text editor and copy and paste the one-liner.
Let’s name the file <em>top-words-1.sh</em> to indicate that this is the first step towards our new command-line tool If you like to stay at the command line, you can use the builtin <code>fc</code>, which stands for <em>fix command</em>, and allows you to fix or <em>edit</em> the last-run command.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">fc</span></pre>
<p>Running <code>fc</code> invokes the default text editor, which is stored in the environment variable <em>EDITOR</em>.
In the Docker container, this is set to <code>nano</code>, a straightforward text editor.
As you can see, this file contains our one-liner:</p>
<pre><span style="background-color: #AAAAAA; color: #000000">  GNU nano 5.4                     /tmp/zshpu0emE                               
</span>curl -sL "https://www.gutenberg.org/files/11/11-0.txt" |                        
tr '[:upper:]' '[:lower:]' |            
grep -oE "[a-z\']{2,}" |                
sort |              
grep -Fvwf stopwords |                  
uniq -c |           
sort -nr |          
head -n 10          
 
 
 
 
                                <span style="background-color: #AAAAAA; color: #000000">[ Read 8 lines ]</span>                                
<span style="background-color: #AAAAAA; color: #000000">^G</span> Help      <span style="background-color: #AAAAAA; color: #000000">^O</span> Write Out <span style="background-color: #AAAAAA; color: #000000">^W</span> Where Is  <span style="background-color: #AAAAAA; color: #000000">^K</span> Cut       <span style="background-color: #AAAAAA; color: #000000">^T</span> Execute   <span style="background-color: #AAAAAA; color: #000000">^C</span> Location    
<span style="background-color: #AAAAAA; color: #000000">^X</span> Exit      <span style="background-color: #AAAAAA; color: #000000">^R</span> Read File <span style="background-color: #AAAAAA; color: #000000">^\</span> Replace   <span style="background-color: #AAAAAA; color: #000000">^U</span> Paste     <span style="background-color: #AAAAAA; color: #000000">^J</span> Justify   <span style="background-color: #AAAAAA; color: #000000">^_</span> Go To Line  </pre>
<p>Let’s give this temporary file a proper name by pressing <strong><code>Ctrl-O</code></strong>, removing the temporary filename, and typing <code>top-words-1.sh</code>:</p>
<pre><span style="background-color: #AAAAAA; color: #000000">  GNU nano 5.4                     /tmp/zshpu0emE                               
</span>curl -sL "https://www.gutenberg.org/files/11/11-0.txt" |                        
tr '[:upper:]' '[:lower:]' |            
grep -oE "[a-z\']{2,}" |                
sort |              
grep -Fvwf stopwords |                  
uniq -c |           
sort -nr |          
head -n 10          
 
 
 
 
<span style="background-color: #AAAAAA; color: #000000">File Name to Write: top-words-1.sh                                              
^G</span> Help             <span style="background-color: #AAAAAA; color: #000000">M-D</span> DOS Format      <span style="background-color: #AAAAAA; color: #000000">M-A</span> Append          <span style="background-color: #AAAAAA; color: #000000">M-B</span> Backup File     
<span style="background-color: #AAAAAA; color: #000000">^C</span> Cancel           <span style="background-color: #AAAAAA; color: #000000">M-M</span> Mac Format      <span style="background-color: #AAAAAA; color: #000000">M-P</span> Prepend         <span style="background-color: #AAAAAA; color: #000000">^T</span> Browse           </pre>
<p>Press <strong><code>Enter</code></strong>:</p>
<pre><span style="background-color: #AAAAAA; color: #000000">  GNU nano 5.4                     /tmp/zshpu0emE                               
</span>curl -sL "https://www.gutenberg.org/files/11/11-0.txt" |                        
tr '[:upper:]' '[:lower:]' |            
grep -oE "[a-z\']{2,}" |                
sort |              
grep -Fvwf stopwords |                  
uniq -c |           
sort -nr |          
head -n 10          
 
 
 
 
<span style="background-color: #AAAAAA; color: #000000">Save file under DIFFERENT NAME?                                                 
 Y</span> Yes                                                                          
<span style="background-color: #AAAAAA; color: #000000"> N</span> No           <span style="background-color: #AAAAAA; color: #000000">^C</span> Cancel                                                       </pre>
<p>Press <strong><code>Y</code></strong> to confirm that you want to save under a different filename:</p>
<pre><span style="background-color: #AAAAAA; color: #000000">  GNU nano 5.4                     top-words-1.sh                               
</span>curl -sL <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> <span style="color: #5f8700">|</span>                        
tr <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">'[:upper:]'</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">'[:lower:]'</span> <span style="color: #5f8700">|</span>            
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">grep</span> -oE <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">"[a-z\']{2,}"</span> <span style="color: #5f8700">|</span>                
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">sort</span> <span style="color: #5f8700">|</span>              
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">grep</span> -Fvwf stopwords <span style="color: #5f8700">|</span>                  
uniq<span style="font-weight: bold"></span><span style="font-weight: bold; color: #af005f"> -c</span> <span style="color: #5f8700">|</span>           
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">sort</span> -nr <span style="color: #5f8700">|</span>          
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">head</span><span style="font-weight: bold; color: #af005f"> -n</span> 10          
 
 
 
 
                               <span style="background-color: #AAAAAA; color: #000000">[ Wrote 8 lines ]</span>                                
<span style="background-color: #AAAAAA; color: #000000">^G</span> Help      <span style="background-color: #AAAAAA; color: #000000">^O</span> Write Out <span style="background-color: #AAAAAA; color: #000000">^W</span> Where Is  <span style="background-color: #AAAAAA; color: #000000">^K</span> Cut       <span style="background-color: #AAAAAA; color: #000000">^T</span> Execute   <span style="background-color: #AAAAAA; color: #000000">^C</span> Location    
<span style="background-color: #AAAAAA; color: #000000">^X</span> Exit      <span style="background-color: #AAAAAA; color: #000000">^R</span> Read File <span style="background-color: #AAAAAA; color: #000000">^\</span> Replace   <span style="background-color: #AAAAAA; color: #000000">^U</span> Paste     <span style="background-color: #AAAAAA; color: #000000">^J</span> Justify   <span style="background-color: #AAAAAA; color: #000000">^_</span> Go To Line  </pre>
<p>Press <strong><code>Ctrl-X</code></strong> to exit <code>nano</code> and go back from whence you came.</p>
<p>We are using the file extension <em>.sh</em> to make clear that we are creating a shell script.
However, command-line tools don’t need to have an extension.
In fact, command-line tools rarely have extensions.</p>
<p>Confirm the contents of the file:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">pwd</span>
/data/ch04
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 44K
-rwxr-xr-x 1 dst dst  400 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">fizzbuzz.py</span>*
-rwxr-xr-x 1 dst dst  391 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">fizzbuzz.R</span>*
-rw-r--r-- 1 dst dst 7.5K Dec 14 11:47 stopwords
-rwxr-xr-x 1 dst dst  182 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">stream.py</span>*
-rwxr-xr-x 1 dst dst  147 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">stream.R</span>*
-rw-r--r-- 1 dst dst  173 Dec 14 11:47 top-words-1.sh
-rwxr-xr-x 1 dst dst  105 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words-4.sh</span>*
-rwxr-xr-x 1 dst dst  128 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words-5.sh</span>*
-rwxr-xr-x 1 dst dst  647 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words.py</span>*
-rwxr-xr-x 1 dst dst  584 Dec 14 11:46 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words.R</span>*
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">top-words-1.sh
</span>───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">top-words-1.sh</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ curl -sL <span style="color: #5f8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> <span style="color: #af005f">|</span>
   2   │ tr <span style="color: #5f8700">'[:upper:]'</span> <span style="color: #5f8700">'[:lower:]'</span> <span style="color: #af005f">|</span>
   3   │ grep -oE <span style="color: #5f8700">"[a-z\']{2,}"</span> <span style="color: #af005f">|</span>
   4   │ sort <span style="color: #af005f">|</span>
   5   │ grep -Fvwf stopwords <span style="color: #af005f">|</span>
   6   │ uniq -c <span style="color: #af005f">|</span>
   7   │ sort -nr <span style="color: #af005f">|</span>
   8   │ head -n 10
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>You can now use <code>bash</code><span class="citation"><a href="#fn51" class="footnote-ref" id="fnref51"><sup>51</sup></a></span> to interpret and execute the commands in the file:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bash</span> <span style="text-decoration: underline">top-words-1.sh</span>
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
     63 king
     60 turtle
     57 mock
     56 hatter
     55 gryphon</pre>
<p>This saves you from typing the one-liner again next time.</p>
<p>However, because the file cannot be executed on its own, it’s not yet a <em>real</em> command-line tool.
Let’s change that in the next step.</p>
</div>
<div id="step-2-give-permission-to-execute" class="section level3" number="4.2.2">
<h3 number="4.2.2"><span class="header-section-number">4.2.2</span> Step 2: Give Permission to Execute</h3>
<p>The reason we cannot execute our file directly is that we don’t have the correct access permissions.
In particular, you, as a user, need to have permission to execute the file.
In this section we change the access permissions of our file.</p>
<p>In order to compare differences between steps, copy the file to <em>top-words-2.sh</em> using <code>cp -v top-words-{1,2}.sh</code>.</p>

<div class="rmdtip">
If you ever want to verify what the brace expansion or any other form of file expansion leads to, replace the command with <code>echo</code> to just print the result.
For example, <code>echo book_{draft,final}.md</code> or <code>echo agent-{001..007}</code>.
</div>
<p>To change the access permissions of a file, we need to use a command-line tool called <code>chmod</code><span class="citation"><a href="#fn52" class="footnote-ref" id="fnref52"><sup>52</sup></a></span>, which stands for <em>change mode</em>.
It changes the file mode bits of a specific file.
The following command gives the user, you, permission to execute <em>top-words-2.sh</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> -v top-words-{1,2}.sh
'top-words-1.sh' -&gt; 'top-words-2.sh'
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">chmod</span> u+x <span style="text-decoration: underline">top-words-2.sh</span></pre>
<p>The argument <code>u+x</code> consists of three characters: (1) <code>u</code> indicates that we want to change the permissions for the user who owns the file, which is you, because you created the file; (2) <code>+</code> indicates that we want to add a permission; and (3) <code>x</code>, which indicates the permissions to execute.</p>
<p>Let’s now have a look at the access permissions of both files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span> top-words-{1,2}.sh
-rw-r--r-- 1 dst dst 173 Dec 14 11:47 top-words-1.sh
-rwxr--r-- 1 dst dst 173 Dec 14 11:47 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">top-words-2.sh</span>*</pre>
<p>The first column shows the access permissions for each file.
For <em>top-words-2.sh</em>, this is <em><code>-rwxrw-r--</code></em>.
The first character <em><code>-</code></em> (hyphen) indicates the file type.
A <em><code>-</code></em> means regular file and a <em><code>d</code></em> means directory.
The next three characters, <em><code>rwx</code></em>, indicate the access permissions for the user who owns the file.
The <em><code>r</code></em> and <em><code>w</code></em> mean <em>read</em> and <em>write</em>, respectively.
(As you can see, <em>top-words-1.sh</em> has a <em><code>-</code></em> instead of an <em><code>x</code></em>, which means that we cannot <em>execute</em> that file.) The next three characters <em><code>rw-</code></em> indicate the access permissions for all members of the group that owns the file.
Finally, the last three characters in the column, <em><code>r--</code></em>, indicate access permissions for all other users.</p>
<p>Now you can execute the file as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">./top-words-2.sh</span>
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
     63 king
     60 turtle
     57 mock
     56 hatter
     55 gryphon</pre>
<p>If you try to execute a file for which you don’t have the correct access permissions, as with <em>top-words-1.sh</em>, you will see the following error message:</p>
<pre><span style="font-weight: bold">$</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">./top-words-1.sh</span>
zsh: permission denied: ./top-words-1.sh</pre>
</div>
<div id="step-3-define-shebang" class="section level3" number="4.2.3">
<h3 number="4.2.3"><span class="header-section-number">4.2.3</span> Step 3: Define Shebang</h3>
<p>Although we can already execute the file on its own, we should add a so-called shebang to the file.
The <em>shebang</em> is a special line in the script that instructs the system which executable it should use to interpret the commands.</p>
<p>The name <em>shebang</em> comes from the first two characters: a hash (she) and an exclamation mark (bang): <code>#!</code>.
It’s not a good idea to leave it out, as we have done in the previous step, because each shell has a different default executable.
The Z shell, the one we’re using throughout the book, uses the executable <em>/bin/sh</em> by default if no shebang is defined.
In this case I’d like <code>bash</code> to interpret the commands as that will give us some more functionality than <code>sh</code>.</p>
<p>Again, you’re free to use whatever editor you like, but I’m going to stick with <code>nano</code><span class="citation"><a href="#fn53" class="footnote-ref" id="fnref53"><sup>53</sup></a></span>, which is installed in the Docker image.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> -v top-words-{2,3}.sh
'top-words-2.sh' -&gt; 'top-words-3.sh'
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">nano</span> <span style="text-decoration: underline">top-words-3.sh</span></pre>
<pre><span style="background-color: #AAAAAA; color: #000000">  GNU nano 5.4                     top-words-3.sh                               
</span>curl -sL <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> <span style="color: #5f8700">|</span>                        
tr <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">'[:upper:]'</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">'[:lower:]'</span> <span style="color: #5f8700">|</span>            
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">grep</span> -oE <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">"[a-z\']{2,}"</span> <span style="color: #5f8700">|</span>                
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">sort</span> <span style="color: #5f8700">|</span>              
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">grep</span> -Fvwf stopwords <span style="color: #5f8700">|</span>                  
uniq<span style="font-weight: bold"></span><span style="font-weight: bold; color: #af005f"> -c</span> <span style="color: #5f8700">|</span>           
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">sort</span> -nr <span style="color: #5f8700">|</span>          
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">head</span><span style="font-weight: bold; color: #af005f"> -n</span> 10          
 
 
 
 
                                <span style="background-color: #AAAAAA; color: #000000">[ Read 8 lines ]</span>                                
<span style="background-color: #AAAAAA; color: #000000">^G</span> Help      <span style="background-color: #AAAAAA; color: #000000">^O</span> Write Out <span style="background-color: #AAAAAA; color: #000000">^W</span> Where Is  <span style="background-color: #AAAAAA; color: #000000">^K</span> Cut       <span style="background-color: #AAAAAA; color: #000000">^T</span> Execute   <span style="background-color: #AAAAAA; color: #000000">^C</span> Location    
<span style="background-color: #AAAAAA; color: #000000">^X</span> Exit      <span style="background-color: #AAAAAA; color: #000000">^R</span> Read File <span style="background-color: #AAAAAA; color: #000000">^\</span> Replace   <span style="background-color: #AAAAAA; color: #000000">^U</span> Paste     <span style="background-color: #AAAAAA; color: #000000">^J</span> Justify   <span style="background-color: #AAAAAA; color: #000000">^_</span> Go To Line  </pre>
<p>Go ahead and type <em><code>#!/usr/bin/env/bash</code></em> and press <strong><code>Enter</code></strong>.
When you’re ready, press <strong><code>Ctrl-X</code></strong> to save and exit.</p>
<pre><span style="background-color: #AAAAAA; color: #000000">  GNU nano 5.4                     top-words-3.sh *                             
</span><span style="color: #00afaf"></span><span style="color: #00afaf">#!/usr/bin/env bash</span><span style="background-color: #5f8700"> 
</span>curl -sL <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> <span style="color: #5f8700">|</span>                        
tr <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">'[:upper:]'</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">'[:lower:]'</span> <span style="color: #5f8700">|</span>            
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">grep</span> -oE <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">"[a-z\']{2,}"</span> <span style="color: #5f8700">|</span>                
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">sort</span> <span style="color: #5f8700">|</span>              
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">grep</span> -Fvwf stopwords <span style="color: #5f8700">|</span>                  
uniq<span style="font-weight: bold"></span><span style="font-weight: bold; color: #af005f"> -c</span> <span style="color: #5f8700">|</span>           
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">sort</span> -nr <span style="color: #5f8700">|</span>          
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">head</span><span style="font-weight: bold; color: #af005f"> -n</span> 10          
 
 
 
<span style="background-color: #AAAAAA; color: #000000">Save modified buffer?                                                           
 Y</span> Yes                                                                          
<span style="background-color: #AAAAAA; color: #000000"> N</span> No           <span style="background-color: #AAAAAA; color: #000000">^C</span> Cancel                                                       </pre>
<p>Press <strong><code>Y</code></strong> to indicate that you want to save the file.</p>
<pre><span style="background-color: #AAAAAA; color: #000000">  GNU nano 5.4                     top-words-3.sh *                             
</span><span style="color: #00afaf"></span><span style="color: #00afaf">#!/usr/bin/env bash</span><span style="background-color: #5f8700"> 
</span>curl -sL <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> <span style="color: #5f8700">|</span>                        
tr <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">'[:upper:]'</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">'[:lower:]'</span> <span style="color: #5f8700">|</span>            
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">grep</span> -oE <span style="font-weight: bold"></span><span style="font-weight: bold; color: #af8700">"[a-z\']{2,}"</span> <span style="color: #5f8700">|</span>                
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">sort</span> <span style="color: #5f8700">|</span>              
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">grep</span> -Fvwf stopwords <span style="color: #5f8700">|</span>                  
uniq<span style="font-weight: bold"></span><span style="font-weight: bold; color: #af005f"> -c</span> <span style="color: #5f8700">|</span>           
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">sort</span> -nr <span style="color: #5f8700">|</span>          
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">head</span><span style="font-weight: bold; color: #af005f"> -n</span> 10          
 
 
 
<span style="background-color: #AAAAAA; color: #000000">File Name to Write: top-words-3.sh                                              
^G</span> Help             <span style="background-color: #AAAAAA; color: #000000">M-D</span> DOS Format      <span style="background-color: #AAAAAA; color: #000000">M-A</span> Append          <span style="background-color: #AAAAAA; color: #000000">M-B</span> Backup File     
<span style="background-color: #AAAAAA; color: #000000">^C</span> Cancel           <span style="background-color: #AAAAAA; color: #000000">M-M</span> Mac Format      <span style="background-color: #AAAAAA; color: #000000">M-P</span> Prepend         <span style="background-color: #AAAAAA; color: #000000">^T</span> Browse           </pre>
<p>Let’s confirm what <em>top-words-3.sh</em> looks like:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">top-words-3.sh
</span>───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">top-words-3.sh</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env bash </span>
   2   │ curl -sL <span style="color: #5f8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> <span style="color: #af005f">|</span>
   3   │ tr <span style="color: #5f8700">'[:upper:]'</span> <span style="color: #5f8700">'[:lower:]'</span> <span style="color: #af005f">|</span>
   4   │ grep -oE <span style="color: #5f8700">"[a-z\']{2,}"</span> <span style="color: #af005f">|</span>
   5   │ sort <span style="color: #af005f">|</span>
   6   │ grep -Fvwf stopwords <span style="color: #af005f">|</span>
   7   │ uniq -c <span style="color: #af005f">|</span>
   8   │ sort -nr <span style="color: #af005f">|</span>
   9   │ head -n 10
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>That’s exactly what we need: our original pipeline with a shebang in front of it.</p>
<p>Sometimes you will come across scripts that have a shebang in the form of <em><code>!/usr/bin/bash</code></em> or <em><code>!/usr/bin/python</code></em> (in the case of Python, as we will see in the next section).
While this generally works, if the <code>bash</code> or <code>python</code><span class="citation"><a href="#fn54" class="footnote-ref" id="fnref54"><sup>54</sup></a></span> executables are installed in a different location than <em>/usr/bin</em>, then the script does not work anymore.
It is better to use the form that I present here, namely <em><code>!/usr/bin/env bash</code></em> and <em><code>!/usr/bin/env python</code></em>, because the <code>env</code><span class="citation"><a href="#fn55" class="footnote-ref" id="fnref55"><sup>55</sup></a></span> executable is aware where <code>bash</code> and <code>python</code> are installed.
In short, using <code>env</code> makes your scripts more portable.</p>
</div>
<div id="step-4-remove-fixed-input" class="section level3" number="4.2.4">
<h3 number="4.2.4"><span class="header-section-number">4.2.4</span> Step 4: Remove Fixed Input</h3>
<p>We know have a valid command-line tool that we can execute from the command line.
But we can do better than this.
We can make our command-line tool more reusable.
The first command in our file is <code>curl</code>, which downloads the text from which we wish to obtain the top 10 most-used words.
So, the data and operations are combined into one.</p>
<p>What if we wanted to obtain the top 10 most-used words from another e-book, or any other text for that matter? The input data is fixed within the tools itself.
It would be better to separate the data from the command-line tool.</p>
<p>If we assume that the user of the command-line tool will provide the text, the tool will become generally applicable.
So, the solution is to remove the <code>curl</code> command from the script.
Here is the updated script named <em>top-words-4.sh</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> -v top-words-{3,4}.sh
'top-words-3.sh' -&gt; 'top-words-4.sh'
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">sed</span> -i <span style="color: #af8700">'2d'</span> <span style="text-decoration: underline">top-words-4.sh</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">top-words-4.sh
</span>───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">top-words-4.sh</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env bash </span>
   2   │ tr <span style="color: #5f8700">'[:upper:]'</span> <span style="color: #5f8700">'[:lower:]'</span> <span style="color: #af005f">|</span>
   3   │ grep -oE <span style="color: #5f8700">"[a-z\']{2,}"</span> <span style="color: #af005f">|</span>
   4   │ sort <span style="color: #af005f">|</span>
   5   │ grep -Fvwf stopwords <span style="color: #af005f">|</span>
   6   │ uniq -c <span style="color: #af005f">|</span>
   7   │ sort -nr <span style="color: #af005f">|</span>
   8   │ head -n 10
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>This works because if a script starts with a command that needs data from standard input, like <code>tr</code>, it will take the input that is given to the command-line tools.
For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">'https://www.gutenberg.org/files/11/11-0.txt'</span> | <span style="color: #5f8700">./top-words-4.sh</span>
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
     63 king
     60 turtle
     57 mock
     56 hatter
     55 gryphon
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">'https://www.gutenberg.org/files/12/12-0.txt'</span> | <span style="color: #5f8700">./top-words-4.sh</span>
    469 alice
    189 queen
     98 gutenberg
     88 project
     72 time
     71 red
     70 white
     67 king
     63 head
     59 knight
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> bash | <span style="color: #5f8700">./top-words-4.sh</span>
    585 command
    332 set
    313 word
    313 option
    304 file
    300 variable
    298 bash
    258 list
    257 expansion
    238 history</pre>

<div class="rmdtip">
Although we have not done so in our script, the same principle holds for saving data.
It is, in general, better to let the user take care of that using output redirection than to let the script write to a specific file.
Of course, if you intend to use a command-line tool only for your own projects, then there are no limits to how specific you can be.
</div>
</div>
<div id="step-5-add-arguments" class="section level3" number="4.2.5">
<h3 number="4.2.5"><span class="header-section-number">4.2.5</span> Step 5: Add Arguments</h3>
<p>There is one more step to make our command-line tool even more reusable: parameters.
In our command-line tool there are a number of fixed command-line arguments, for example <code>-nr</code> for <code>sort</code> and <code>-n 10</code> for <code>head</code>.
It is probably best to keep the former argument fixed.
However, it would be very useful to allow for different values for the <code>head</code> command.
This would allow the end user to set the number of most-often used words to output.
Below shows what our file <em>top-words-5.sh</em> looks like:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">top-words-5.sh
</span>───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">top-words-5.sh</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env bash</span>
   2   │
   3   │ NUM_WORDS<span style="color: #af005f">=</span><span style="color: #5f8700">"${1</span><span style="color: #af005f">:-</span><span style="color: #5f8700">10}"</span>
   4   │
   5   │ tr <span style="color: #5f8700">'[:upper:]'</span> <span style="color: #5f8700">'[:lower:]'</span> <span style="color: #af005f">|</span>
   6   │ grep -oE <span style="color: #5f8700">"[a-z\']{2,}"</span> <span style="color: #af005f">|</span>
   7   │ sort <span style="color: #af005f">|</span>
   8   │ grep -Fvwf stopwords <span style="color: #af005f">|</span>
   9   │ uniq -c <span style="color: #af005f">|</span>
  10   │ sort -nr <span style="color: #af005f">|</span>
  11   │ head -n <span style="color: #5f8700">"${NUM_WORDS}"</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<ul>
<li>The variable <em>NUM_WORDS</em> is set to the value of <em>$1</em>, which is a special variable in Bash. It holds the value of the first command-line argument passed to our command-line tool. The table below lists the other special variables that Bash offers. If no value is specified, it will take on the value “10.”</li>
<li>Note that in order to <em>use</em> the value of the <em>$NUM_WORDS</em> variable, you need to put a dollar sign in front of it. When you <em>set</em> it, you don’t write a dollar sign.</li>
</ul>
<p>We could have also used <em>$1</em> directly as an argument for <code>head</code> and not bother creating an extra variable such <em>NUM_WORDS</em>.
However, with larger scripts and a few more command-line arguments such as <em>$2</em> and <em>$3</em>, your code becomes more readable when you use named variables.</p>
<p>Now if you wanted to see the top 20 most-used words of our text, we would invoke our command-line tool as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> <span style="color: #af8700">&gt;</span> alice.txt
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">./top-words-5.sh</span> 20
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
     63 king
     60 turtle
     57 mock
     56 hatter
     55 gryphon
     53 rabbit
     50 head
     48 voice
     45 looked
     44 mouse
     42 duchess
     40 tone
     40 dormouse
     37 cat
     34 march</pre>
<p>If the user does not specify a number, then our script will show the top 10 most common words:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">./top-words-5.sh</span>
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
     63 king
     60 turtle
     57 mock
     56 hatter
     55 gryphon</pre>
</div>
<div id="step-6-extend-your-path" class="section level3" number="4.2.6">
<h3 number="4.2.6"><span class="header-section-number">4.2.6</span> Step 6: Extend Your PATH</h3>
<p>After the previous five steps we are finally finished building a reusable command-line tool.
There is, however, one more step that can be very useful.
In this optional step we are going to ensure that you can execute your command-line tools from everywhere.</p>
<p>Currently, when you want to execute your command-line tool, you either have to navigate to the directory it is in or include the full path name as shown in step 2.
This is fine if the command-line tool is specifically built for, say, a certain project.
However, if your command-line tool could be applied in multiple situations, then it is useful to be able to execute it from everywhere, just like the command-line tools that come with Ubuntu.</p>
<p>To accomplish this, Bash needs to know where to look for your command-line tools.
It does this by traversing a list of directories which are stored in an environment variable called <em>PATH</em>.
In a fresh Docker container, the <em>PATH</em> looks like this:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> $PATH
/usr/local/lib/R/site-library/rush/exec:/usr/bin/dsutils:/home/dst/.local/bin:/u
sr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin</pre>
<p>The directories are delimited by colons.
We can print it as a list of directories by <em>translating</em> the colons to newlines:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> $PATH | <span style="color: #5f8700">tr</span> <span style="color: #af8700">':'</span> <span style="color: #af8700">'\n'</span>
/usr/local/lib/R/site-library/rush/exec
/usr/bin/dsutils
/home/dst/.local/bin
/usr/local/sbin
/usr/local/bin
/usr/sbin
/usr/bin
/sbin
/bin</pre>
<p>To change the <em>PATH</em> permanently, you’ll need to edit the <em>.bashrc</em> or <em>.profile</em> file located in your home directory.
If you put all your custom command-line tools into one directory, say, <em>~/tools</em>, then you only change the <em>PATH</em> once.
Now, you no longer need to add the <em>./</em>, but you can just use the filename.
Moreover, you do no longer need to remember where the command-line tool is located.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> -v top-words{-5.sh,}
'top-words-5.sh' -&gt; 'top-words'
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">export</span> PATH=<span style="color: #af8700">"</span><span style="color: #00afaf">${PATH}</span><span style="color: #af8700">:/data/ch04"</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> $PATH
/usr/local/lib/R/site-library/rush/exec:/usr/bin/dsutils:/home/dst/.local/bin:/u
sr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/data/ch04
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> <span style="color: #af8700">"https://www.gutenberg.org/files/11/11-0.txt"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">top-words</span> 10
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  170k  100  170k    0     0   115k      0  0:00:01  0:00:01 --:--:--  116k
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
     63 king
     60 turtle
     57 mock
     56 hatter
     55 gryphon</pre>
</div>
</div>
<div id="creating-command-line-tools-with-python-and-r" class="section level2" number="4.3">
<h2 number="4.3"><span class="header-section-number">4.3</span> Creating Command-line Tools with Python and R</h2>
<p>The command-line tool that we created in the previous section was written in Bash.
(Sure, not every feature of the Bash programming language was employed, but the interpreter still was <code>bash</code>.) As you know by now, the command line is language agnostic, so we don’t necessarily have to use Bash for creating command-line tools.</p>
<p>In this section I’m going demonstrate that command-line tools can be created in other programming languages as well.
I’ll focus on Python and R because these are the two most popular programming languages within the data science community.
I cannot offer a complete introduction to either language, so I assume that you have some familiarity with Python and or R.
Other programming languages such as Java, Go, and Julia, follow a similar pattern when it comes to creating command-line tools.</p>
<p>There are three main reasons for creating command-line tools in another programming language than Bash.
First, you may already have some code that you’d like to be able to use from the command line.
Second, the command-line tool would end up encompassing more than a hundred lines of Bash code.
Third, the command-line tool needs to be more safe and robust (Bash lacks many features such as type checking).</p>
<p>The six steps that I discussed in the previous section roughly apply to creating command-line tools in other programming languages as well.
The first step, however, would not be copy pasting from the command line, but rather copy pasting the relevant code into a new file.
Command-line tools written in Python and R need to specify <code>python</code> and <code>Rscript</code><span class="citation"><a href="#fn56" class="footnote-ref" id="fnref56"><sup>56</sup></a></span>, respectively, as the interpreter after the shebang.</p>
<p>When it comes to creating command-line tools using Python and R, there are two more aspects that deserve special attention.
First, processing standard input, which comes natural to shell scripts, has to be taken care of explicitly in Python and R.
Second, as command-line tools written in Python and R tend to be more complex, we may also want to offer the user the ability to specify more elaborate command-line arguments.</p>
<div id="porting-the-shell-script" class="section level3" number="4.3.1">
<h3 number="4.3.1"><span class="header-section-number">4.3.1</span> Porting The Shell Script</h3>
<p>As a starting point, let’s see how we would port the shell script we just created to both Python and R.
In other words, what Python and R code gives us the top most-often used words from standard input? We will first show the two files <em>top-words.py</em> and <em>top-words.R</em> and then discuss the differences with the shell code.
In Python, the code would look something like:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch04</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">top-words.py</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">top-words.py</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env python</span>
   2   │ <span style="color: #af005f">import</span> re
   3   │ <span style="color: #af005f">import</span> sys
   4   │
   5   │ <span style="color: #af005f">from</span> collections <span style="color: #af005f">import</span> Counter
   6   │ <span style="color: #af005f">from</span> urllib.request <span style="color: #af005f">import</span> urlopen
   7   │
   8   │ <span style="color: #af005f">def</span> <span style="color: #0087ff">top_words</span>(text, n):
   9   │     <span style="color: #af005f">with</span> urlopen(<span style="color: #5f8700">"https://raw.githubusercontent.com/stopwords-iso/stopw
</span>       │ <span style="color: #5f8700">ords-en/master/stopwords-en.txt"</span>) <span style="color: #af005f">as</span> f:
  10   │         stopwords <span style="color: #af005f">=</span> f.read().decode(<span style="color: #5f8700">"utf-8"</span>).split(<span style="color: #5f8700">"</span><span style="color: #00afaf">\n</span><span style="color: #5f8700">"</span>)
  11   │
  12   │     words <span style="color: #af005f">=</span> re.findall(<span style="color: #5f8700">"[a-z']{2,}"</span>, text.lower())
  13   │     words <span style="color: #af005f">=</span> (w <span style="color: #af005f">for</span> w <span style="color: #af005f">in</span> words <span style="color: #af005f">if</span> w <span style="color: #af005f">not</span> <span style="color: #af005f">in</span> stopwords)
  14   │
  15   │     <span style="color: #af005f">for</span> word, count <span style="color: #af005f">in</span> Counter(words).most_common(n):
  16   │         <span style="color: #00afaf">print</span>(<span style="color: #af005f">f</span><span style="color: #5f8700">"</span>{count<span style="color: #af8700">:&gt;7</span>}<span style="color: #5f8700"> </span>{word}<span style="color: #5f8700">"</span>)
  17   │
  18   │
  19   │ <span style="color: #af005f">if</span> __name__ <span style="color: #af005f">==</span> <span style="color: #5f8700">"__main__"</span>:
  20   │     text <span style="color: #af005f">=</span> sys.stdin.read()
  21   │
  22   │     <span style="color: #af005f">try</span>:
  23   │         n <span style="color: #af005f">=</span> int(sys.argv[<span style="color: #af8700">1</span>])
  24   │     <span style="color: #af005f">except</span>:
  25   │         n <span style="color: #af005f">=</span> <span style="color: #af8700">10</span>
  26   │
  27   │     top_words(text, n)
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Note that this Python example doesn’t use any third-party packages.
If you want to do advanced text processing, then I recommend you check out the NLTK package<span class="citation"><a href="#fn57" class="footnote-ref" id="fnref57"><sup>57</sup></a></span>.
If you’re going to work with a lot of numerical data, then I recommend you use the Pandas package<span class="citation"><a href="#fn58" class="footnote-ref" id="fnref58"><sup>58</sup></a></span>.</p>
<p>And in R the code would look something like:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">top-words.R</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">top-words.R</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env Rscript</span>
   2   │ n <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">as.integer</span>(<span style="color: #00afaf">commandArgs</span>(trailingOnly <span style="color: #af005f">=</span> <span style="color: #af8700">TRUE</span>))
   3   │ <span style="color: #af005f">if</span> (<span style="color: #00afaf">length</span>(n) <span style="color: #af005f">==</span> <span style="color: #af8700">0</span>) n <span style="color: #af005f">&lt;-</span> <span style="color: #af8700">10</span>
   4   │
   5   │ f_stopwords <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">url</span>(<span style="color: #5f8700">"https://raw.githubusercontent.com/stopwords-iso/sto
</span>       │ <span style="color: #5f8700">pwords-en/master/stopwords-en.txt"</span>)
   6   │ stopwords <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">readLines</span>(f_stopwords, warn <span style="color: #af005f">=</span> <span style="color: #af8700">FALSE</span>)
   7   │ <span style="color: #00afaf">close</span>(f_stopwords)
   8   │
   9   │ f_text <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">file</span>(<span style="color: #5f8700">"stdin"</span>)
  10   │ lines <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">tolower</span>(<span style="color: #00afaf">readLines</span>(f_text))
  11   │
  12   │ words <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">unlist</span>(<span style="color: #00afaf">regmatches</span>(lines, <span style="color: #00afaf">gregexpr</span>(<span style="color: #5f8700">"[a-z']{2,}"</span>, lines)))
  13   │ words <span style="color: #af005f">&lt;-</span> words[<span style="color: #00afaf">is.na</span>(<span style="color: #00afaf">match</span>(words, stopwords))]
  14   │
  15   │ counts <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">sort</span>(<span style="color: #00afaf">table</span>(words), decreasing <span style="color: #af005f">=</span> <span style="color: #af8700">TRUE</span>)
  16   │ <span style="color: #00afaf">cat</span>(<span style="color: #00afaf">sprintf</span>(<span style="color: #5f8700">"%7d %s</span><span style="color: #00afaf">\n</span><span style="color: #5f8700">"</span>, counts[<span style="color: #af8700">1</span><span style="color: #af005f">:</span>n], <span style="color: #00afaf">names</span>(counts[<span style="color: #af8700">1</span><span style="color: #af005f">:</span>n])), sep <span style="color: #af005f">=</span> <span style="color: #5f8700">""</span>)
  17   │ <span style="color: #00afaf">close</span>(f_text)
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Let’s check that all three implementations (i.e., Bash, Python, and R) return the same top 5 words with the same counts:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">time</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">top-words</span> 5
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
top-words 5 &lt; alice.txt  0.38s user 0.04s system 149% cpu 0.278 total
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">time</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">top-words.py</span> 5
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
top-words.py 5 &lt; alice.txt  1.36s user 0.03s system 95% cpu 1.454 total
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">time</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">top-words.R</span> 5
    403 alice
     98 gutenberg
     88 project
     76 queen
     71 time
top-words.R 5 &lt; alice.txt  1.48s user 0.14s system 91% cpu 1.774 total</pre>
<p>Wonderful! Sure, the output itself is not very exciting.
What’s exciting is that we can accomplish the same task with multiple languages.
Let’s look at the differences between the approaches.</p>
<p>First, what’s immediately obvious is the difference in amount of code.
For this specific task, both Python and R require much more code than Bash.
This illustrates that, for some tasks, it is better to use the command line.
For other tasks, you may better off using a programming language.
As you gain more experience on the command line, you will start to recognize when to use which approach.
When everything is a command-line tool, you can even split up the task into subtasks, and combine a Bash command-line tool with, say, a Python command-line tool.
Whichever approach works best for the task at hand.</p>
</div>
<div id="processing-streaming-data-from-standard-input" class="section level3" number="4.3.2">
<h3 number="4.3.2"><span class="header-section-number">4.3.2</span> Processing Streaming Data from Standard Input</h3>
<p>In the previous two code snippets, both Python and R read the complete standard input at once.
On the command line, most tools pipe data to the next command-line tool in a streaming fashion.
There are a few command-line tools which require the complete data before they write any data to standard output, like <code>sort</code>.
This means the pipeline is blocked by such command-line tools.
This doesn’t have to be a problem when the input data is finite, like a file.
However, when the input data is a non-stop stream, such blocking command-line tools are useless.</p>
<p>Luckily Python and R support processing streaming data.
You can apply a function on a line-per-line basis, for example.
Here are two minimal examples that demonstrate how this works in Python and R, respectively.</p>
<p>Both the Python and R tool solve the, by now infamous, Fizz Buzz problem, which is defined as follows: Print the numbers from 1 to 100, except that if the number is divisible by 3, instead print “fizz”; if the number is divisible by 5, instead print “buzz”; and if the number is divisible by 15, instead print “fizzbuzz.” Here’s the Python code<a href="#fn59" class="footnote-ref" id="fnref59"><sup>59</sup></a>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">fizzbuzz.py</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">fizzbuzz.py</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env python</span>
   2   │ <span style="color: #af005f">import</span> sys
   3   │
   4   │ CYCLE_OF_15 <span style="color: #af005f">=</span> [<span style="color: #5f8700">"fizzbuzz"</span>, <span style="color: #af8700">None</span>, <span style="color: #af8700">None</span>, <span style="color: #5f8700">"fizz"</span>, <span style="color: #af8700">None</span>,
   5   │                <span style="color: #5f8700">"buzz"</span>, <span style="color: #5f8700">"fizz"</span>, <span style="color: #af8700">None</span>, <span style="color: #af8700">None</span>, <span style="color: #5f8700">"fizz"</span>,
   6   │                <span style="color: #5f8700">"buzz"</span>, <span style="color: #af8700">None</span>, <span style="color: #5f8700">"fizz"</span>, <span style="color: #af8700">None</span>, <span style="color: #af8700">None</span>]
   7   │
   8   │ <span style="color: #af005f">def</span> <span style="color: #0087ff">fizz_buzz</span>(n: int) -&gt; str:
   9   │     <span style="color: #af005f">return</span> CYCLE_OF_15[n <span style="color: #af005f">%</span> <span style="color: #af8700">15</span>] <span style="color: #af005f">or</span> str(n)
  10   │
  11   │ <span style="color: #af005f">if</span> __name__ <span style="color: #af005f">==</span> <span style="color: #5f8700">"__main__"</span>:
  12   │     <span style="color: #af005f">try</span>:
  13   │         <span style="color: #af005f">while</span> (n<span style="color: #af005f">:=</span> sys.stdin.readline()):
  14   │             <span style="color: #00afaf">print</span>(fizz_buzz(int(n)))
  15   │     <span style="color: #af005f">except</span>:
  16   │         <span style="color: #af005f">pass</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>And here’s the R code:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">fizzbuzz.R</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">fizzbuzz.R</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env Rscript</span>
   2   │ cycle_of_15 <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">c</span>(<span style="color: #5f8700">"fizzbuzz"</span>, <span style="color: #af8700">NA</span>, <span style="color: #af8700">NA</span>, <span style="color: #5f8700">"fizz"</span>, <span style="color: #af8700">NA</span>,
   3   │                  <span style="color: #5f8700">"buzz"</span>, <span style="color: #5f8700">"fizz"</span>, <span style="color: #af8700">NA</span>, <span style="color: #af8700">NA</span>, <span style="color: #5f8700">"fizz"</span>,
   4   │                  <span style="color: #5f8700">"buzz"</span>, <span style="color: #af8700">NA</span>, <span style="color: #5f8700">"fizz"</span>, <span style="color: #af8700">NA</span>, <span style="color: #af8700">NA</span>)
   5   │
   6   │ <span style="color: #0087ff">fizz_buzz</span> <span style="color: #af005f">&lt;-</span> <span style="color: #af005f">function</span>(n) {
   7   │   word <span style="color: #af005f">&lt;-</span> cycle_of_15[<span style="color: #00afaf">as.integer</span>(n) <span style="color: #af005f">%%</span> <span style="color: #af8700">15</span> <span style="color: #af005f">+</span> <span style="color: #af8700">1</span>]
   8   │   <span style="color: #00afaf">ifelse</span>(<span style="color: #00afaf">is.na</span>(word), n, word)
   9   │ }
  10   │
  11   │ f <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">file</span>(<span style="color: #5f8700">"stdin"</span>)
  12   │ <span style="color: #00afaf">open</span>(f)
  13   │ <span style="color: #af005f">while</span>(<span style="color: #00afaf">length</span>(n <span style="color: #af005f">&lt;-</span> <span style="color: #00afaf">readLines</span>(f, n <span style="color: #af005f">=</span> <span style="color: #af8700">1</span>)) <span style="color: #af005f">&gt;</span> <span style="color: #af8700">0</span>) {
  14   │   <span style="color: #00afaf">write</span>(fizz_buzz(n), <span style="color: #00afaf">stdout</span>())
  15   │ }
  16   │ <span style="color: #00afaf">close</span>(f)
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Let’s test both tools (to save space I pipe the output to <code>column</code>):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 30 | <span style="color: #5f8700">fizzbuzz.py</span> | <span style="color: #5f8700">column</span> -x
1               2               fizz            4               buzz
fizz            7               8               fizz            buzz
11              fizz            13              14              fizzbuzz
16              17              fizz            19              buzz
fizz            22              23              fizz            buzz
26              fizz            28              29              fizzbuzz
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 30 | <span style="color: #5f8700">fizzbuzz.R</span> | <span style="color: #5f8700">column</span> -x
1               2               fizz            4               buzz
fizz            7               8               fizz            buzz
11              fizz            13              14              fizzbuzz
16              17              fizz            19              buzz
fizz            22              23              fizz            buzz
26              fizz            28              29              fizzbuzz</pre>
<p>This output looks correct to me!
It’s difficult to demonstrate that these two tools actually work in a streaming manner.
You can verify this yourself by piping the input data to <code>sample -d 100</code> before it’s piped to the Python or R tool.
That way, you’ll add a small delay in between each line so that it’s easier to confirm that the tools don’t wait for all the input data, operate on a line by line basis.</p>
</div>
</div>
<div id="summary-3" class="section level2" number="4.4">
<h2 number="4.4"><span class="header-section-number">4.4</span> Summary</h2>
<p>In this intermezzo chapter, I have shown you how to build your own command-line tool.
Only six steps are needed to turn your code into a reusable building block.
You’ll find that it makes you much more productive.
I advise you to keep an eye out for opportunities to create your own tools.
The next chapter covers the second step of the OSEMN model for data science, namely scrubbing data.</p>
</div>
<div id="for-further-exploration-3" class="section level2" number="4.5">
<h2 number="4.5"><span class="header-section-number">4.5</span> For Further Exploration</h2>
<ul>
<li>Adding help documentation to your tool becomes important when the tool has many options to remember, and even more so when you want to share your tool with others. <code>docopt</code> is a language-agnostic framework to provide help and define the possible options that your tool accepts. There are implementations available in just about any programming language including Bash, Python, and R.</li>
<li>If you want to learn more about programming in Bash, I recommend <em>Classic Shell Programming</em> by Arnold Robbins and Nelson Beebe and <em>Bash Cookbook</em> by Carl Albing and JP Vossen.</li>
<li>Writing a robust and safe Bash script is quite tricky. <a href="https://www.shellcheck.net/">ShellCheck</a> is an online tool that will check your Bash code for mistakes and vulnerabilities. There’s also a command-line tool available.</li>
<li>The book <em>Ten Essays on Fizz Buzz</em> by Joel Grus is an insightful and fun collection of ten different ways to solve Fizz Buzz with Python.</li>
</ul>
<!--chapter:end:04.Rmd-->
</div>
</div>
<div id="chapter-5-scrubbing-data" class="section level1" number="5">
<h1 number="5"><span class="header-section-number">5</span> Scrubbing Data</h1>
<p>Two chapters ago, in the first step of the OSEMN model for data science, we looked at <em>obtaining</em> data from a variety of sources.
This chapter is all about the second step: <em>scrubbing</em> data.
You see, it’s quite rare that you can immediately continue with <em>exploring</em> or even <em>modeling</em> the data.
There’s a plethora of reasons why your data first needs some cleaning, or scrubbing.</p>
<p>For starters, the data might not be in the desired format.
For example, you may have obtained some JSON data from an API, but you need it to be in CSV format to create a visualization.
Other common formats include plain text, HTML, and XML.
Most command-line tools only work with one or two formats, so it’s important that you’re able to convert data from one format to another.</p>
<p>Once the data is in the desired format, there could still be issues like missing values, inconsistencies, weird characters, or unnecessary parts.
You can fix these by applying filters, replacing values, and combining multiple files.
The command line is especially well-suited for these kind of transformations, because there are many specialized tools available, most of which can handle large amounts of data.
In this chapter I’ll discuss classic tools such as <code>grep</code><span class="citation"><a href="#fn60" class="footnote-ref" id="fnref60"><sup>60</sup></a></span> and <code>awk</code><span class="citation"><a href="#fn61" class="footnote-ref" id="fnref61"><sup>61</sup></a></span>, and newer tools such as <code>jq</code><span class="citation"><a href="#fn62" class="footnote-ref" id="fnref62"><sup>62</sup></a></span> and <code>pup</code><span class="citation"><a href="#fn63" class="footnote-ref" id="fnref63"><sup>63</sup></a></span>.</p>
<p>Sometimes you can use the same command-line tool to perform several operations or multiple tools to perform the same operation.
This chapter is more structured like a cookbook, where the focus is on the problems or recipes, rather than diving deeply into the command-line tools themselves.</p>
<div id="overview-2" class="section level2" number="5.1">
<h2 number="5.1"><span class="header-section-number">5.1</span> Overview</h2>
<!-- #TODO: SHOULD: Review the list below once the chapter is complete -->
<p>In this chapter, you’ll learn how to:</p>
<ul>
<li>Convert data from one format to another</li>
<li>Apply SQL queries directly to CSV</li>
<li>Filter lines</li>
<li>Extract and replace values</li>
<li>Split, merge, and extract columns</li>
<li>Combine multiple files</li>
</ul>
<p>This chapter starts with the following files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch05</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 200K
-rw-r--r-- 1 dst dst 164K Dec 14 11:47 alice.txt
-rw-r--r-- 1 dst dst 4.5K Dec 14 11:47 iris.csv
-rw-r--r-- 1 dst dst  179 Dec 14 11:47 irismeta.csv
-rw-r--r-- 1 dst dst  160 Dec 14 11:47 names-comma.csv
-rw-r--r-- 1 dst dst  129 Dec 14 11:47 names.csv
-rw-r--r-- 1 dst dst 7.8K Dec 14 11:47 tips.csv
-rw-r--r-- 1 dst dst 5.1K Dec 14 11:47 users.json</pre>
<p>The instructions to get these files are in <a href="#chapter-2-getting-started">Chapter 2</a>.
Any other files are either downloaded or generated using command-line tools.</p>
<p>Before I dive into the actual transformations, I’d like to demonstrate their ubiquity when working at the command line.</p>
</div>
<div id="transformations-transformations-everywhere" class="section level2" number="5.2">
<h2 number="5.2"><span class="header-section-number">5.2</span> Transformations, Transformations Everywhere</h2>
<p>In <a href="#chapter-1-introduction">Chapter 1</a> I mentioned that, in practice, the steps of the OSEMN model will rarely be followed linearly.
In this vein, although scrubbing is the second step of the OSEMN model, I want you to know that it’s not <em>just</em> the obtained data that needs scrubbing.
The transformations that you’ll learn in this chapter can be useful at any part of your pipeline and at any step of the OSEMN model.
Generally, if one command line tool generates output that can be used immediately by the next tool, you can chain the two tools together by using the pipe operator (<code>|</code>).
Otherwise, a transformation needs to be applied to the data first by inserting an intermediate tool into the pipeline.</p>
<p>Let me walk you through an example to make this more concrete.
Imagine that you have obtained the first 100 items of a <em>fizzbuzz</em> sequence (cf. <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>) and that you’d like to visualize how often the words <em>fizz</em>, <em>buzz</em>, and <em>fizzbuzz</em> appear using a bar chart.
Don’t worry if this example uses tools that you might not be familiar with yet, they’ll all be covered in more detail later.</p>
<p>First you obtain the data by generating the sequence and write it to <em>fb.seq</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 100 |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">/data/ch04/fizzbuzz.py</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tee</span> fb.seq | <span style="color: #5f8700">trim</span>
1
2
fizz
4
buzz
fizz
7
8
fizz
buzz
… with 90 more lines</pre>
<p><span class="callout">&#10122;</span> The custom tool <code>fizzbuzz.py</code> comes from <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>.</p>
<p>Then you use <code>grep</code> to keep the lines that match the pattern <em>fizz</em> or <em>buzz</em> and count how often each word appears using <code>sort</code> and <code>uniq</code><span class="citation"><a href="#fn64" class="footnote-ref" id="fnref64"><sup>64</sup></a></span>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">grep</span> -E <span style="color: #af8700">"fizz|buzz"</span> <span style="text-decoration: underline">fb.seq</span> | <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> | <span style="color: #5f8700">uniq</span> -c | <span style="color: #5f8700">sort</span> -nr <span style="color: #af8700">&gt;</span> fb.cnt <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10123;</span></span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> -A <span style="text-decoration: underline">fb.cnt</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">fb.cnt</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #00afaf">·····</span>27<span style="color: #00afaf">·</span>fizz<span style="color: #af005f">␊</span>
   2   │ <span style="color: #00afaf">·····</span>14<span style="color: #00afaf">·</span>buzz<span style="color: #af005f">␊</span>
   3   │ <span style="color: #00afaf">······</span>6<span style="color: #00afaf">·</span>fizzbuzz<span style="color: #af005f">␊</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p><span class="callout">&#10122;</span> This regular expression also matches <em>fizzbuzz</em>.
<br><span class="callout">&#10123;</span> Using <code>sort</code> and <code>uniq</code> this way is a common way to count lines and sort them in descending order. It’s the <code>-c</code> option that adds the counts.</p>
<p>Note that <code>sort</code> is used twice: first because <code>uniq</code> assumes its input data to be sorted and second to sort the counts numerically.
In a way, this is an intermediate transformation, albeit a subtle one.</p>
<p>The next step would be to visualize the counts using <code>rush</code><span class="citation"><a href="#fn65" class="footnote-ref" id="fnref65"><sup>65</sup></a></span>.
However, since <code>rush</code> expects the input data to be in CSV format, this requires a less subtle transformation first.
<code>awk</code> can add a header, flip the two fields, and insert commas in a single incantation:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">fb.cnt</span> <span style="color: #5f8700">awk</span> <span style="color: #af8700">'BEGIN { print "value,count" } { print $2","$1 }'</span> <span style="color: #af8700">&gt;</span> fb.csv
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">fb.csv</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">fb.csv</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #af005f">value,count
</span>   2   │ <span style="color: #af005f">fizz,</span><span style="color: #af8700">27</span>
   3   │ <span style="color: #af005f">buzz,</span><span style="color: #af8700">14</span>
   4   │ <span style="color: #af005f">fizzbuzz,</span><span style="color: #af8700">6</span>
───────┴────────────────────────────────────────────────────────────────────────
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> <span style="text-decoration: underline">fb.csv</span>
│ value    │ count │
├──────────┼───────┤
│ fizz     │    27 │
│ buzz     │    14 │
│ fizzbuzz │     6 │</pre>
<p>Now you’re ready to use <code>rush</code> to create a bar chart.
See Figure @ref(fig:fb-image) for the result.
(I’ll cover this syntax of <code>rush</code> in detail in <a href="#chapter-7-exploring-data">Chapter 7</a>.)</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot -x value -y count --geom col --height 2 <span style="text-decoration: underline">fb.csv</span> <span style="color: #af8700">&gt;</span> fb.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">fb.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/fb.png" alt="Counting fizz, buzz, and fizzbuzz" width="90%" />
<p class="caption">
(#fig:fb-image)Counting fizz, buzz, and fizzbuzz
</p>
</div>
<p>Although this example is a bit contrived, it reveals a pattern that is common when working at the command line.
The key tools, such as the ones that obtain data, create a visualization, or train a model, often require intermediate transformations in order to be chained into a pipeline.
In that sense, writing a pipeline is like solving a puzzle, where the key pieces often require helper pieces to fit.</p>
<p>Now that you’ve seen the importance of scrubbing data, you’re ready to learn about some actual transformations.</p>
</div>
<div id="plain-text" class="section level2" number="5.3">
<h2 number="5.3"><span class="header-section-number">5.3</span> Plain Text</h2>
<p>Formally speaking, <em>plain text</em> refers to a sequence of human-readable characters and optionally, some specific types of control characters such as tabs and newlines<span class="citation"><a href="#fn66" class="footnote-ref" id="fnref66"><sup>66</sup></a></span>.
Examples are logs, e-books, emails, and source code.
Plain text has many benefits over binary data<span class="citation"><a href="#fn67" class="footnote-ref" id="fnref67"><sup>67</sup></a></span>, including:</p>
<ul>
<li>It can be opened, edited, and saved using any text editor</li>
<li>It’s self-describing and independent of the application that created it</li>
<li>It will outlive other forms of data, because no additional knowledge or applications are required to process it</li>
</ul>
<p>But most importantly, the Unix philosophy considers plain text to be the universal interface between command-line tools<span class="citation"><a href="#fn68" class="footnote-ref" id="fnref68"><sup>68</sup></a></span>.
Meaning, most tools accept plain text as input and produce plain text as output.</p>
<p>That’s reason enough for me to start with plain text.
The other formats that I discuss in this chapter, CSV, JSON, XML, and HTML are indeed also plain text.
For now, I assume that the plain text has no clear tabular structure (like CSV does) or nested structure (like JSON, XML, and HTML do).
Later in this chapter, I’ll introduce some tools that are specifically designed for working with these formats.</p>
<!-- #TODO: SHOULD: What to do with this part? -->
<!-- Although the tools in this section can also be applied to these other formats (because they're still text), -->
<!-- keep in mind that the tools treat the data as plain text, and don't interpret the tabular or nested structure. -->
<!-- Sometimes you can get away with this,  -->
<div id="filtering-lines" class="section level3" number="5.3.1">
<h3 number="5.3.1"><span class="header-section-number">5.3.1</span> Filtering Lines</h3>
<p>The first scrubbing operation is filtering lines.
This means that from the input data, each line will be evaluated whether it will be kept or discarded.</p>
<div id="based-on-location" class="section level4" number="5.3.1.1">
<h4 number="5.3.1.1"><span class="header-section-number">5.3.1.1</span> Based on Location</h4>
<p>The most straightforward way to filter lines is based on their location.
This may be useful when you want to inspect, say, the top 10 lines of a file, or when you extract a specific row from the output of another command-line tool.
To illustrate how to filter based on location, let’s create a dummy file that contains 10 lines:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> -f <span style="color: #af8700">"Line %g"</span> 10 | <span style="color: #5f8700">tee</span> lines
Line 1
Line 2
Line 3
Line 4
Line 5
Line 6
Line 7
Line 8
Line 9
Line 10</pre>
<p>You can print the first 3 lines using either <code>head</code><span class="citation"><a href="#fn69" class="footnote-ref" id="fnref69"><sup>69</sup></a></span>, <code>sed</code><span class="citation"><a href="#fn70" class="footnote-ref" id="fnref70"><sup>70</sup></a></span>, or <code>awk</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">head</span> -n 3
Line 1
Line 2
Line 3
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">sed</span> -n <span style="color: #af8700">'1,3p'</span>
Line 1
Line 2
Line 3
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">awk</span> <span style="color: #af8700">'NR &lt;= 3'</span> <span class="callout">&#10122;</span>
Line 1
Line 2
Line 3</pre>
<p><span class="callout">&#10122;</span> In <code>awk</code>, <em>NR</em> refers to the total number of input records seen so far.</p>
<p>Similarly, you can print the last 3 lines using <code>tail</code><span class="citation"><a href="#fn71" class="footnote-ref" id="fnref71"><sup>71</sup></a></span>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">tail</span> -n 3
Line 8
Line 9
Line 10</pre>
<p>You can also you use <code>sed</code> and <code>awk</code> for this, but <code>tail</code> is much faster.
Removing the first 3 lines goes as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">tail</span> -n +4
Line 4
Line 5
Line 6
Line 7
Line 8
Line 9
Line 10
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">sed</span> <span style="color: #af8700">'1,3d'
</span>Line 4
Line 5
Line 6
Line 7
Line 8
Line 9
Line 10
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">sed</span> -n <span style="color: #af8700">'1,3!p'</span>
Line 4
Line 5
Line 6
Line 7
Line 8
Line 9
Line 10</pre>
<p>Notice that with <code>tail</code> you have to specify the number of lines plus one.
Think of it as the line from which you want to start printing.
Removing the last 3 lines can be done with <code>head</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">head</span> -n -3
Line 1
Line 2
Line 3
Line 4
Line 5
Line 6
Line 7</pre>
<p>You can print specific lines using a either <code>sed</code>, <code>awk</code>, or a combination of <code>head</code> and <code>tail</code>.
Here I print lines 4, 5, and 6:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">sed</span> -n <span style="color: #af8700">'4,6p'</span>
Line 4
Line 5
Line 6
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">awk</span> <span style="color: #af8700">'(NR&gt;=4) &amp;&amp; (NR&lt;=6)'</span>
Line 4
Line 5
Line 6
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">head</span> -n 6 | <span style="color: #5f8700">tail</span> -n 3
Line 4
Line 5
Line 6</pre>
<p>You can print odd lines with <code>sed</code> by specifying a start and a step, or with <code>awk</code> by using the modulo operator:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">sed</span> -n <span style="color: #af8700">'1~2p'</span>
Line 1
Line 3
Line 5
Line 7
Line 9
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">awk</span> <span style="color: #af8700">'NR%2'
</span>Line 1
Line 3
Line 5
Line 7
Line 9</pre>
<p>Printing even lines works in a similar manner:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">sed</span> -n <span style="color: #af8700">'0~2p'</span>
Line 2
Line 4
Line 6
Line 8
Line 10
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">lines</span> <span style="color: #5f8700">awk</span> <span style="color: #af8700">'(NR+1)%2'</span>
Line 2
Line 4
Line 6
Line 8
Line 10</pre>
<!-- #TODO: SHOULD: Mention somewhere in the book that Linux doesn't care about file extensions -->

<div class="rmdnote">
Many of these examples start with the smaller-than sign (<code>&lt;</code>) followed by the filename.
I do this because this allows me to read the pipeline from left to right.
Please know that this is my own preference.
You can also use <code>cat</code> to pipe the contents of a file.
Also, many command-line tools also accept the filename as an argument.
</div>
</div>
<div id="based-on-a-pattern" class="section level4" number="5.3.1.2">
<h4 number="5.3.1.2"><span class="header-section-number">5.3.1.2</span> Based on a Pattern</h4>
<p>Sometimes you want to keep or discard lines based on their contents.
With <code>grep</code>, the canonical command-line tool for filtering lines, you can print every line that matches a certain pattern or regular expression.
For example, to extract all the chapter headings from <em>Alice’s Adventures in Wonderland</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">grep</span> -i chapter <span class="callout">&#10122;</span>
CHAPTER I. Down the Rabbit-Hole
CHAPTER II. The Pool of Tears
CHAPTER III. A Caucus-Race and a Long Tale
CHAPTER IV. The Rabbit Sends in a Little Bill
CHAPTER V. Advice from a Caterpillar
CHAPTER VI. Pig and Pepper
CHAPTER VII. A Mad Tea-Party
CHAPTER VIII. The Queen's Croquet-Ground
CHAPTER IX. The Mock Turtle's Story
CHAPTER X. The Lobster Quadrille
CHAPTER XI. Who Stole the Tarts?
CHAPTER XII. Alice's Evidence</pre>
<p><span class="callout">&#10122;</span> The <code>-i</code> options specifies that the matching should be case-insensitive.</p>
<p>You can also specify a regular expression.
For example, if you only wanted to print the headings that start with <em>The</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">grep</span> -E <span style="color: #af8700">'^CHAPTER (.*)\. The'</span>
CHAPTER II. The Pool of Tears
CHAPTER IV. The Rabbit Sends in a Little Bill
CHAPTER VIII. The Queen's Croquet-Ground
CHAPTER IX. The Mock Turtle's Story
CHAPTER X. The Lobster Quadrille</pre>
<p>Note that you have to specify the <code>-E</code> option in order to enable regular expressions.
Otherwise, <code>grep</code> interprets the pattern as a literal string which most likely results in no matches at all:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">grep</span> <span style="color: #af8700">'^CHAPTER (.*)\. The'</span></pre>
<p>With the <code>-v</code> option you invert the matches, so that <code>grep</code> prints the lines which <em>don’t</em> match the pattern.
The regular expression below matches lines that contain white space, only.
So with the inverse, and using <code>wc -l</code>, you can count the number of non-empty lines:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">grep</span> -Ev <span style="color: #af8700">'^\s$'</span> | <span style="color: #5f8700">wc</span> -l
2790</pre>
</div>
<div id="based-on-randomness" class="section level4" number="5.3.1.3">
<h4 number="5.3.1.3"><span class="header-section-number">5.3.1.3</span> Based on Randomness</h4>
<p>When you’re in the process of formulating your data pipeline and you have a lot of data, then debugging your pipeline can be cumbersome.
In that case, generating a smaller sample from the data might be useful.
This is where <code>sample</code><span class="citation"><a href="#fn72" class="footnote-ref" id="fnref72"><sup>72</sup></a></span> comes in handy.
The main purpose of <code>sample</code> is to get a subset of the data by outputting only a certain percentage of the input on a line-by-line basis.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> -f <span style="color: #af8700">"Line %g"</span> 1000 | <span style="color: #5f8700">sample</span> -r 1%
Line 45
Line 223
Line 355
Line 369
Line 438
Line 807
Line 813</pre>
<p>Here, every input line has a one percent chance of being printed.
This percentage can also be specified as a fraction (namely <code>1/100</code>) or as a probability (namely <code>0.01</code>).</p>
<p><code>sample</code> has two other purposes, which can be useful when you’re debugging your pipeline.
First, it’s possible to add some delay to the output.
This comes in handy when the input is a constant stream (for example, the Wikipedia stream we saw in <a href="#chapter-3-obtaining-data">Chapter 3</a>), and the data comes in too fast to see what’s going on.
Secondly, you can put a timer on <code>sample</code>, so that you don’t have to kill the ongoing process manually.
For example, to add a 1 second delay between each line being printed and to only run for 5 seconds, you would type:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> -f <span style="color: #af8700">"Line %g"</span> 1000 | <span style="color: #5f8700">sample</span> -r 1% -d 1000 -s 5 | <span style="color: #5f8700">ts</span> <span class="callout">&#10122;</span>
Dec 14 11:48:10 Line 58
Dec 14 11:48:11 Line 60
Dec 14 11:48:12 Line 230
Dec 14 11:48:13 Line 250
Dec 14 11:48:14 Line 785
Dec 14 11:48:15 Line 786</pre>
<p><span class="callout">&#10122;</span> The tool <code>ts</code><span class="citation"><a href="#fn73" class="footnote-ref" id="fnref73"><sup>73</sup></a></span> adds a timestamp in front of each line.</p>
<p>In order to prevent unnecessary computation, try to put <code>sample</code> as early as possible in your pipeline.
In fact, this argument holds for any command-line tool that reduces data, like <code>head</code> and <code>tail</code>.
Once you’re confident your pipeline works, you take it out of the pipeline.</p>
</div>
</div>
<div id="extracting-values" class="section level3" number="5.3.2">
<h3 number="5.3.2"><span class="header-section-number">5.3.2</span> Extracting Values</h3>
<p>To extract the actual chapter headings from our example earlier, you can take a simple approach by piping the output of <code>grep</code> to <code>cut</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">grep</span> -i chapter <span style="text-decoration: underline">alice.txt</span> | <span style="color: #5f8700">cut</span> -d <span style="color: #af8700">' '</span> -f 3-
Down the Rabbit-Hole
The Pool of Tears
A Caucus-Race and a Long Tale
The Rabbit Sends in a Little Bill
Advice from a Caterpillar
Pig and Pepper
A Mad Tea-Party
The Queen's Croquet-Ground
The Mock Turtle's Story
The Lobster Quadrille
Who Stole the Tarts?
Alice's Evidence</pre>
<p>Here, each line that’s passed to <code>cut</code> is being split on spaces into fields, and then the third field to the last field is being printed.
The total number of fields can be different per input line.
With <code>sed</code> you can accomplish the same task in a much more complex manner:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">sed</span> -rn <span style="color: #af8700">'s/^CHAPTER ([IVXLCDM]{1,})\. (.*)$/\2/p'</span> <span style="text-decoration: underline">alice.txt</span> | <span style="color: #5f8700">trim</span> 3
Down the Rabbit-Hole
The Pool of Tears
A Caucus-Race and a Long Tale
… with 9 more lines</pre>
<p>(Since the output is the same it’s trimmed to three lines.) This approach uses a regular expression and a back reference.
Here, <code>sed</code> also takes over the work done by <code>grep</code>.
I only recommend using such a complicated approach when a simpler one would not work.
For example, if <em>chapter</em> was ever part of the text itself and not just used to indicate the start of a new chapter.
Of course there are many levels of complexity which would have worked around this, but this is to illustrate an extremely strict approach.
In practice, the challenge is to come up with a pipeline that strikes a good balance between complexity and flexibility.</p>
<p>It’s worth noting that <code>cut</code> can also split on characters positions.
This is useful for when you want to extract (or remove) the same set of characters per input line:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">grep</span> -i chapter <span style="text-decoration: underline">alice.txt</span> | <span style="color: #5f8700">cut</span> -c 9-
I. Down the Rabbit-Hole
II. The Pool of Tears
III. A Caucus-Race and a Long Tale
IV. The Rabbit Sends in a Little Bill
V. Advice from a Caterpillar
VI. Pig and Pepper
VII. A Mad Tea-Party
VIII. The Queen's Croquet-Ground
IX. The Mock Turtle's Story
X. The Lobster Quadrille
XI. Who Stole the Tarts?
XII. Alice's Evidence</pre>
<p><code>grep</code> has a great feature that outputs every match onto a separate line using the <code>-o</code> option:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">grep</span> -oE <span style="color: #af8700">'\w{2,}'</span> | <span style="color: #5f8700">trim</span>
Project
Gutenberg
Alice
Adventures
in
Wonderland
by
Lewis
Carroll
This
… with 28615 more lines</pre>
<p>But what if you wanted to create a dataset of all the words that start with an <em>a</em> and end with an <em>e</em>?
Well, of course there’s a pipeline for that too:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">alice.txt</span> <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[:upper:]'</span> <span style="color: #af8700">'[:lower:]'</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">grep</span> -oE <span style="color: #af8700">'\w{2,}'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">grep</span> -E <span style="color: #af8700">'^a.*e$'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> | <span style="color: #5f8700">uniq</span> | <span style="color: #5f8700">sort</span> -nr | <span style="color: #5f8700">trim</span>
available
ate
assistance
askance
arise
argue
are
archive
applicable
apple
… with 25 more lines</pre>
<p><span class="callout">&#10122;</span> Here I use <code>tr</code> to make the text lowercase. We’ll have a closer look at <code>tr</code> in the next section.</p>
<p>The two <code>grep</code> commands might have been combined into one, but in this case I decided it would be easier to reuse and adapt the previous pipeline.
There’s no shame in being pragmatic in order to get the job done!</p>
</div>
<div id="replacing-and-deleting-values" class="section level3" number="5.3.3">
<h3 number="5.3.3"><span class="header-section-number">5.3.3</span> Replacing and Deleting Values</h3>
<p>You can use the command-line tool <code>tr</code><span class="citation"><a href="#fn74" class="footnote-ref" id="fnref74"><sup>74</sup></a></span>, which stands for <em>translate</em>, to replace or delete individual characters.
For example, spaces can be replaced by underscores as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hello world!'</span> | <span style="color: #5f8700">tr</span> <span style="color: #af8700">' '</span> <span style="color: #af8700">'_'</span>
hello_world!</pre>
<p>If more than one character needs to be replaced, then you can combine that:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hello world!'</span> | <span style="color: #5f8700">tr</span> <span style="color: #af8700">' !'</span> <span style="color: #af8700">'_?'</span>
hello_world?</pre>
<p><code>tr</code> can also be used to delete individual characters by specifying the argument <code>-d</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hello world!'</span> | <span style="color: #5f8700">tr</span> -d <span style="color: #af8700">' !'</span>
helloworld
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hello world!'</span> | <span style="color: #5f8700">tr</span> -d -c <span style="color: #af8700">'[a-z]'
</span>helloworld<span style="font-weight: bold; font-style: italic">%</span></pre>
<p>In this case, these two commands accomplish the same thing.
The second command, however, uses two additional features:
It specifies a <em>range</em> of characters (all lowercase letters) using the square brackets and the dash (<code>[-]</code>), and the <code>-c</code> option indicates that complement of that should be used.
In other words, this command only keeps lowercase letters.
You can even use <code>tr</code> to convert text to uppercase:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hello world!'</span> | <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[a-z]'</span> <span style="color: #af8700">'[A-Z]'</span>
HELLO WORLD!
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hello world!'</span> | <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[:lower:]'</span> <span style="color: #af8700">'[:upper:]'</span>
HELLO WORLD!</pre>
<p>However, if you need to translate non-ASCII characters, then <code>tr</code> may not work because it operates on single-byte characters only. In those cases you should use <code>sed</code> instead:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hello world!'</span> | <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[a-z]'</span> <span style="color: #af8700">'[A-Z]'</span>
HELLO WORLD!
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hallo wêreld!'</span> | <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[a-z]'</span> <span style="color: #af8700">'[A-Z]'</span>
HALLO WêRELD!
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hallo wêreld!'</span> | <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[:lower:]'</span> <span style="color: #af8700">'[:upper:]'</span>
HALLO WêRELD!
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'hallo wêreld!'</span> | <span style="color: #5f8700">sed</span> <span style="color: #af8700">'s/[[:lower:]]*/\U&amp;/g'</span>
HALLO WÊRELD!
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'helló világ'</span> | <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[:lower:]'</span> <span style="color: #af8700">'[:upper:]'</span>
HELLó VILáG
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'helló világ'</span> | <span style="color: #5f8700">sed</span> <span style="color: #af8700">'s/[[:lower:]]*/\U&amp;/g'</span>
HELLÓ VILÁG</pre>
<!-- #TODO: SHOULD: Give a proper intro about sed -->
<p>If you need to operate on more than individual characters, then you may find <code>sed</code> useful.
You’ve already seen an example of <code>sed</code> with extracting the chapter headings from <em>alice.txt</em>.
Extracting, deleting, and replacing is actually all the same operation in <code>sed</code>.
You just specify different regular expressions.
For example, to change a word, remove repeated spaces, and remove leading spaces:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">' hello     world!'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sed</span> -re <span style="color: #af8700">'s/hello/bye/'</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sed</span> -re <span style="color: #af8700">'s/\s+/ /g'</span> | <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sed</span> -re <span style="color: #af8700">'s/\s+//'</span> <span class="callout">&#10124;</span>
bye world!</pre>
<p><span class="callout">&#10122;</span> Replace <em>hello</em> with <em>bye</em>.
<br><span class="callout">&#10123;</span> Replace any whitespace with one space. The flag <code>g</code> stands for global, meaning that the same substitution can be applied more than once on the same line.
<br><span class="callout">&#10124;</span> This only removes leading spaces because I didn’t specify the flag <code>g</code> here.</p>
<p>Again, just as with the <code>grep</code> example earlier, these three <code>sed</code> commands can be combined into one:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">' hello     world!'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sed</span> -re <span style="color: #af8700">'s/hello/bye/;s/\s+/ /g;s/\s+//'</span>
bye world!</pre>
<p>But tell me, what do you find easier to read?</p>
</div>
</div>
<div id="csv" class="section level2" number="5.4">
<h2 number="5.4"><span class="header-section-number">5.4</span> CSV</h2>
<!-- #TODO: MUST: ### Canonical Format -->
<!-- The `csvkit` documentation[@csvkit] opens with: [...] CSV, the king of tabular file formats. -->
<!-- Canonical format. The format that we want to convert to. -->
<!-- Two properties. it's plain text and it has a rectangular shape, meaning in consists of rows and columns. -->
<!-- As a format, -->
<!-- As a shape, -->
<!-- Even kings have their flaws. -->
<!-- CSV definitely has its flaws. No meta data. No rules regarding quoting, the delimiter, header. -->
<!-- Writing a robust CSV parser is really hard. Nevertheless, CSV has many advantages. -->
<!-- CSV, which is the main format I'll be working with in this chapter, is actually not the easiest format to work with. Many CSV datasets are broken or incompatible with each other because there is no standard syntax, unlike XML and JSON. -->
<!-- Export from Database, spreadsheets, rectangular. -->
<!-- - It's still plain text, so can be read, edited, and even created in any text editor. -->
<!-- - CSV can be imported by many programming languages such as Python, R, and JavaScript and many software such as Excel, Tableau, and Power BI. -->
<!-- - Related to the previous point, but worth mentioning separately: a CSV directly translates to a data frame in Python (with the pandas package), R, and Julia. This means that can immediately continue scrubbing, exploring, and modeling in those languages. -->
<!-- - Speaking of modeling, most machine learning algorithms expect data to be in a rectangular format, or more precisely, a matrix of numerical values where each row is a data point and each column is a feature. More on this in Chapter 9. -->
<!-- makes assumptions, same number of fields per row. -->
<!-- flat. not nested. -->
<!-- CVS is not nested structure -->
<!-- If you build an API, use JSON -->
<!-- If you -->
<!-- You rarely have to covert CSV to anything else when you're doing data analysis. -->
<div id="bodies-and-headers-and-columns-oh-my" class="section level3" number="5.4.1">
<h3 number="5.4.1"><span class="header-section-number">5.4.1</span> Bodies and Headers and Columns, Oh My!</h3>
<p>The command-line tools that I’ve used to scrub plain text, such as <code>tr</code> and <code>grep</code>, cannot always be applied to CSV.
The reason is that these command-line tools have no notion of headers, bodies, and columns.
What if you want to filter lines using <code>grep</code> but always include the header in the output?
Or what if you only want to uppercase the values of a specific column using <code>tr</code> and leave the other columns untouched?</p>
<p>There are multi-step workarounds for this, but they are very cumbersome.
I have something better.
In order to leverage ordinary command-line tools for CSV, I’d like to introduce you to three command-line tools, aptly named: <code>body</code><span class="citation"><a href="#fn75" class="footnote-ref" id="fnref75"><sup>75</sup></a></span>, <code>header</code><span class="citation"><a href="#fn76" class="footnote-ref" id="fnref76"><sup>76</sup></a></span>, and <code>cols</code><span class="citation"><a href="#fn77" class="footnote-ref" id="fnref77"><sup>77</sup></a></span>.</p>
<p>Let’s start with the first command-line tool, <code>body</code>.
With <code>body</code> you can apply any command-line tool to the body of a CSV file, that is, everything excluding the header.
For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> -e <span style="color: #af8700">"value\n7\n2\n5\n3"</span> | <span style="color: #5f8700">body</span> sort -n
value
2
3
5
7</pre>
<p>It assumes that the header of the CSV file only spans one row.
It works like this:</p>
<ul>
<li>Take one line from standard in and store it as a variable named <em>$header</em>.</li>
<li>Print out the header.</li>
<li>Execute all the command-line arguments passed to <code>body</code> on the remaining data in standard in.</li>
</ul>
<p>Here’s another example.
Imagine that you want to count the lines of the following CSV file:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">header</span> -a count
count
1
2
3
4
5</pre>
<p>With <code>wc -l</code>, you can count the number of all lines:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">header</span> -a count | <span style="color: #5f8700">wc</span> -l
6</pre>
<p>If you only want to consider the lines in the body (so everything except the header), you add <code>body</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">header</span> -a count | <span style="color: #5f8700">body</span> wc -l
count
5</pre>
<p>Note that the header is not used and is also printed again in the output.</p>
<p>The second command-line tool, <code>header</code> allows you to manipulate the header of a CSV file.
If no arguments are provided, the header of the CSV file is printed:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #5f8700">header</span>
bill,tip,sex,smoker,day,time,size</pre>
<p>This is the same as <code>head -n 1</code>.
If the header spans more than one row, which is not recommended, you can specify <code>-n 2</code>.
You can also add a header to a CSV file:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">header</span> -a count
count
1
2
3
4
5</pre>
<p>This is equivalent to <code>echo "count" | cat - &lt;(seq 5)</code>.
Deleting a header is done with the <code>-d</code> option:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">iris.csv</span> <span style="color: #5f8700">header</span> -d | <span style="color: #5f8700">trim</span>
5.1,3.5,1.4,0.2,Iris-setosa
4.9,3.0,1.4,0.2,Iris-setosa
4.7,3.2,1.3,0.2,Iris-setosa
4.6,3.1,1.5,0.2,Iris-setosa
5.0,3.6,1.4,0.2,Iris-setosa
5.4,3.9,1.7,0.4,Iris-setosa
4.6,3.4,1.4,0.3,Iris-setosa
5.0,3.4,1.5,0.2,Iris-setosa
4.4,2.9,1.4,0.2,Iris-setosa
4.9,3.1,1.5,0.1,Iris-setosa
… with 140 more lines</pre>
<p>This is similar to <code>tail -n +2</code>, but it’s a bit easier to remember.
Replacing a header, which is basically first deleting a header and then adding one if you look at the above source code, is accomplished by specifying the <code>-r</code> option. Here, we combine it with <code>body</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">header</span> -a line | <span style="color: #5f8700">body</span> wc -l | <span style="color: #5f8700">header</span> -r count
count
5</pre>
<p>And last but not least, you can apply a command to just the header, similar to what the <code>body</code> command-line tool does to the body.
For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">header</span> -a line | <span style="color: #5f8700">header</span> -e <span style="color: #af8700">"tr '[a-z]' '[A-Z]'"</span>
LINE
1
2
3
4
5</pre>
<p>The third command-line tool is called <code>cols</code>, which allows you to apply a certain command to only a subset of the columns.
For example, if you wanted to uppercase the values in the day column in the tips data set (without affecting the other columns and the header), you would use <code>cols</code> in combination with <code>body</code>, as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #5f8700">cols</span> -c day body <span style="color: #af8700">"tr '[a-z]' '[A-Z]'"</span> | <span style="color: #5f8700">head</span> -n 5 | <span style="color: #5f8700">csvlook</span>
│        day │  bill │  tip │ sex    │ smoker │ time   │ size │
├────────────┼───────┼──────┼────────┼────────┼────────┼──────┤
│ 0001-01-07 │ 16.99 │ 1.01 │ Female │  False │ Dinner │    2 │
│ 0001-01-07 │ 10.34 │ 1.66 │ Male   │  False │ Dinner │    3 │
│ 0001-01-07 │ 21.01 │ 3.50 │ Male   │  False │ Dinner │    3 │
│ 0001-01-07 │ 23.68 │ 3.31 │ Male   │  False │ Dinner │    2 │</pre>
<p>Please note that passing multiple command-line tools and arguments as command to <code>header -e</code>, <code>body</code>, and <code>cols</code> can lead to tricky quoting citations.
If you ever run in such problems, it’s best to create a separate command-line tool for this and pass that as command.</p>
<p>In conclusion, while it is generally preferable to use command-line tools which are specifically made for CSV data, <code>body</code>, <code>header</code>, and <code>cols</code> also allow you to apply the classic command-line tools to CSV files if needed.</p>
</div>
<div id="performing-sql-queries-on-csv" class="section level3" number="5.4.2">
<h3 number="5.4.2"><span class="header-section-number">5.4.2</span> Performing SQL Queries on CSV</h3>
<p>In case the command-line tools mentioned in this chapter do not provide enough flexibility, then there is another approach to scrub your data from the command line.
The tool <code>csvsql</code><span class="citation"><a href="#fn78" class="footnote-ref" id="fnref78"><sup>78</sup></a></span> allows you to execute SQL queries directly on CSV files.
SQL is a powerful language to define operations for scrubbing data; it’s a very different way than using individual command-line tools.</p>

<div class="rmdnote">
If your data originally comes from a relational database, then, if possible, try to execute SQL queries on that database and subsequently extract the data as CSV. As I discussed in <a href="#chapter-3-obtaining-data">Chapter 3</a>, you can use the command-line tool <code>sql2csv</code> for this. When you first export data from the database to a CSV file, and then apply SQL, it is not only slower, but there is also a possibility that the column types are not correctly inferred from the CSV data.
</div>
<p>In the scrubbing tasks below, I’ll include several solutions that involve <code>csvsql</code>. A basic command is this:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">header</span> -a val | <span style="color: #5f8700">csvsql</span> --query <span style="color: #af8700">"SELECT SUM(val) AS sum FROM stdin"</span>
sum
15.0</pre>
<p>If you pass standard input to <code>csvsql</code>, then the table is named <em>stdin</em>.
The types of the column are automatically inferred from the data.
As you’ll see later, in the combining CSV files section, you can also specify multiple CSV files.
<!-- #TODO: MUST: Reference SQLite -->
Please keep in mind that <code>csvsql</code> employs SQLite dialect of SQL, which has some subtle differences with respect to the SQL standard.
While SQL is generally more verbose than the other solutions, it is also much more flexible.
If you already know how to tackle a scrubbing problem with SQL, then why not use it when you’re at the command line?</p>
</div>
<div id="extracting-and-reordering-columns" class="section level3" number="5.4.3">
<h3 number="5.4.3"><span class="header-section-number">5.4.3</span> Extracting and Reordering Columns</h3>
<!-- #TODO: MUST: Replace csvcut with xsv select -->
<p>Columns can be extracted and reordered using the command-line tool: <code>csvcut</code><span class="citation"><a href="#fn79" class="footnote-ref" id="fnref79"><sup>79</sup></a></span>.
For example, to keep only the columns in the Iris data set that contain numerical values <em>and</em> reorder the middle two columns:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">iris.csv</span> <span style="color: #5f8700">csvcut</span> -c sepal_length,petal_length,sepal_width,petal_width | <span style="color: #5f8700">csvlo
ok</span>
│ sepal_length │ petal_length │ sepal_width │ petal_width │
├──────────────┼──────────────┼─────────────┼─────────────┤
│          5.1 │          1.4 │         3.5 │         0.2 │
│          4.9 │          1.4 │         3.0 │         0.2 │
│          4.7 │          1.3 │         3.2 │         0.2 │
│          4.6 │          1.5 │         3.1 │         0.2 │
│          5.0 │          1.4 │         3.6 │         0.2 │
│          5.4 │          1.7 │         3.9 │         0.4 │
│          4.6 │          1.4 │         3.4 │         0.3 │
│          5.0 │          1.5 │         3.4 │         0.2 │
… with 142 more lines</pre>
<p>Alternatively, you can also specify the columns you want to leave out with the <code>-C</code> option, which stands for <em>complement</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">iris.csv</span> <span style="color: #5f8700">csvcut</span> -C species | <span style="color: #5f8700">csvlook
</span>│ sepal_length │ sepal_width │ petal_length │ petal_width │
├──────────────┼─────────────┼──────────────┼─────────────┤
│          5.1 │         3.5 │          1.4 │         0.2 │
│          4.9 │         3.0 │          1.4 │         0.2 │
│          4.7 │         3.2 │          1.3 │         0.2 │
│          4.6 │         3.1 │          1.5 │         0.2 │
│          5.0 │         3.6 │          1.4 │         0.2 │
│          5.4 │         3.9 │          1.7 │         0.4 │
│          4.6 │         3.4 │          1.4 │         0.3 │
│          5.0 │         3.4 │          1.5 │         0.2 │
… with 142 more lines</pre>
<p>Here, the included columns are kept in the same order.
Instead of the column names, you can also specify the indices of the columns, which start at 1.
This allows you to, for example, select only the odd columns (should you ever need it!):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'a,b,c,d,e,f,g,h,i\n1,2,3,4,5,6,7,8,9'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvcut</span> -c <span style="color: #af005f">$(</span><span style="color: #5f8700">seq</span> 1 2 9 | <span style="color: #5f8700">paste</span> -sd,<span style="color: #af005f">)</span>
a,c,e,g,i
1,3,5,7,9</pre>
<p>If you’re certain that there are no comma’s in any of the values, then you can also use <code>cut</code> to extract columns.
Be aware that <code>cut</code> does not reorder columns, as is demonstrated with the following command:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'a,b,c,d,e,f,g,h,i\n1,2,3,4,5,6,7,8,9'</span> | <span style="color: #5f8700">cut</span> -d, -f 5,1,3
a,c,e
1,3,5</pre>
<p>As you can see, it does not matter in which order you specify the columns with the <code>-f</code> option; with <code>cut</code> they will always appear in the original order.
For completeness, let’s also take a look at the SQL approach for extracting and reordering the numerical columns of the Iris data set:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">iris.csv</span> <span style="color: #5f8700">csvsql</span> --query <span style="color: #af8700">"SELECT sepal_length, petal_length, "</span>\
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">"sepal_width, petal_width FROM stdi</span>n<span style="color: #5f8700">" | </span>head <span style="color: #5f8700">-n 5 | </span>csvlook
│ sepal_length │ petal_length │ sepal_width │ petal_width │
├──────────────┼──────────────┼─────────────┼─────────────┤
│          5.1 │          1.4 │         3.5 │         0.2 │
│          4.9 │          1.4 │         3.0 │         0.2 │
│          4.7 │          1.3 │         3.2 │         0.2 │
│          4.6 │          1.5 │         3.1 │         0.2 │</pre>
</div>
<div id="filtering-rows" class="section level3" number="5.4.4">
<h3 number="5.4.4"><span class="header-section-number">5.4.4</span> Filtering Rows</h3>
<p>The difference between filtering rows in a CSV file as opposed to filtering lines in a plain text file is that you may want to base this filtering on values in a certain column, only.
Filtering on location is essentially the same, but you have to take into account that the first line of a CSV file is usually the header.
Remember that you can always use the <code>body</code> command-line tool if you want to keep the header:</p>
<!-- #TODO: #MUST use xsv slice and xsv search -->
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">sed</span> -n <span style="color: #af8700">'3,5p'</span>
3
4
5
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">header</span> -a count | <span style="color: #5f8700">body</span> sed -n <span style="color: #af8700">'3,5p'</span>
count
3
4
5</pre>
<p>When it comes down to filtering on a certain pattern within a certain column, you can use either <code>csvgrep</code><span class="citation"><a href="#fn80" class="footnote-ref" id="fnref80"><sup>80</sup></a></span>, <code>awk</code>, or, of course, <code>csvsql</code>.
For example, to exclude all the bills of which the party size was smaller than 5:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvgrep</span> -c size -i -r <span style="color: #af8700">"[1-4]"</span> <span style="text-decoration: underline">tips.csv
</span>bill,tip,sex,smoker,day,time,size
29.8,4.2,Female,No,Thur,Lunch,6
34.3,6.7,Male,No,Thur,Lunch,6
41.19,5.0,Male,No,Thur,Lunch,5
27.05,5.0,Female,No,Thur,Lunch,6
29.85,5.14,Female,No,Sun,Dinner,5
48.17,5.0,Male,No,Sun,Dinner,6
20.69,5.0,Male,No,Sun,Dinner,5
30.46,2.0,Male,Yes,Sun,Dinner,5
28.15,3.0,Male,Yes,Sat,Dinner,5</pre>
<p>Both <code>awk</code> and <code>csvsql</code> can also do numerical comparisons.
For example, to get all the bills above 40 USD on a Saturday or a Sunday:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #5f8700">awk</span> -F, <span style="color: #af8700">'NR==1 || ($1 &gt; 40.0) &amp;&amp; ($5 ~ /^S/)'</span>
bill,tip,sex,smoker,day,time,size
48.27,6.73,Male,No,Sat,Dinner,4
44.3,2.5,Female,Yes,Sat,Dinner,3
48.17,5.0,Male,No,Sun,Dinner,6
50.81,10.0,Male,Yes,Sat,Dinner,3
45.35,3.5,Male,Yes,Sun,Dinner,3
40.55,3.0,Male,Yes,Sun,Dinner,2
48.33,9.0,Male,No,Sat,Dinner,4</pre>
<p>The <code>csvsql</code> solution is more verbose but it’s also more robust as it uses the names of the columns instead of their indexes:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvsql</span> --query <span style="color: #af8700">"SELECT * FROM tips WHERE bill &gt; 40 AND day LIKE 'S%'"</span> <span style="text-decoration: underline">tips.csv
</span>
bill,tip,sex,smoker,day,time,size
48.27,6.73,Male,0,Sat,Dinner,4.0
44.3,2.5,Female,1,Sat,Dinner,3.0
48.17,5.0,Male,0,Sun,Dinner,6.0
50.81,10.0,Male,1,Sat,Dinner,3.0
45.35,3.5,Male,1,Sun,Dinner,3.0
40.55,3.0,Male,1,Sun,Dinner,2.0
48.33,9.0,Male,0,Sat,Dinner,4.0</pre>
<p>Note that the flexibility of the <em>WHERE</em> clause in an SQL query cannot be easily matched with other command-line tools, because SQL can operate on dates and sets, and form complex combinations of clauses.</p>
</div>
<div id="merging-columns" class="section level3" number="5.4.5">
<h3 number="5.4.5"><span class="header-section-number">5.4.5</span> Merging Columns</h3>
<p>Merging columns is useful for when the values of interest are spread over multiple columns.
This may happen with dates (where year, month, and day could be separate columns) or names (where the first name and last name are separate columns).
Let’s consider the second situation.</p>
<!-- #TODO: Must use composers.csv -->
<p>The input CSV is a list of composers.
Imagine that your task is to combine the first name and the last name into a full name.
I’ll present four different approaches for this task: <code>sed</code>, <code>awk</code>, <code>cols</code> + <code>tr</code>, and <code>csvsql</code>.
Let’s have a look at the input CSV:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> -I <span style="text-decoration: underline">names.csv</span>
│ id │ last_name │ first_name │ born │
├────┼───────────┼────────────┼──────┤
│ 1  │ Williams  │ John       │ 1932 │
│ 2  │ Elfman    │ Danny      │ 1953 │
│ 3  │ Horner    │ James      │ 1953 │
│ 4  │ Shore     │ Howard     │ 1946 │
│ 5  │ Zimmer    │ Hans       │ 1957 │</pre>
<p>The first approach, <code>sed</code>, uses two statements.
The first is to replace the header and the second is a regular expression with back references applied to the second row onwards:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names.csv</span> <span style="color: #5f8700">sed</span> -re <span style="color: #af8700">'1s/.*/id,full_name,born/g;2,$s/(.*),(.*),(.*),(.*)/\1,\3
\2,\4/g'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvlook</span> -I
│ id │ full_name     │ born │
├────┼───────────────┼──────┤
│ 1  │ John Williams │ 1932 │
│ 2  │ Danny Elfman  │ 1953 │
│ 3  │ James Horner  │ 1953 │
│ 4  │ Howard Shore  │ 1946 │
│ 5  │ Hans Zimmer   │ 1957 │</pre>
<p>The <code>awk</code> approach looks as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names.csv</span> <span style="color: #5f8700">awk</span> -F, <span style="color: #af8700">'BEGIN{OFS=","; print "id,full_name,born"} {if(NR &gt; 1) {pr
int $1,$3" "$2,$4}}'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvlook</span> -I
│ id │ full_name     │ born │
├────┼───────────────┼──────┤
│ 1  │ John Williams │ 1932 │
│ 2  │ Danny Elfman  │ 1953 │
│ 3  │ James Horner  │ 1953 │
│ 4  │ Howard Shore  │ 1946 │
│ 5  │ Hans Zimmer   │ 1957 │</pre>
<p>The <code>cols</code> approach in combination with <code>tr</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names.csv</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">|</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">cols</span> -c first_name,last_name tr \",\" \" \" |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">header</span> -r full_name,id,born |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvcut</span> -c id,full_name,born |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvlook</span> -I
│ id │ full_name     │ born │
├────┼───────────────┼──────┤
│ 1  │ John Williams │ 1932 │
│ 2  │ Danny Elfman  │ 1953 │
│ 3  │ James Horner  │ 1953 │
│ 4  │ Howard Shore  │ 1946 │
│ 5  │ Hans Zimmer   │ 1957 │</pre>
<p>Please note that <code>csvsql</code> employ SQLite as the database to execute the query and that <code>||</code> stands for concatenation:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names.csv</span> <span style="color: #5f8700">csvsql</span> --query <span style="color: #af8700">"SELECT id, first_name || ' ' || last_name "</span>\
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">"AS full_name, born FROM stdi</span>n<span style="color: #5f8700">" | csv</span>look -I
│ id  │ full_name     │ born   │
├─────┼───────────────┼────────┤
│ 1.0 │ John Williams │ 1932.0 │
│ 2.0 │ Danny Elfman  │ 1953.0 │
│ 3.0 │ James Horner  │ 1953.0 │
│ 4.0 │ Howard Shore  │ 1946.0 │
│ 5.0 │ Hans Zimmer   │ 1957.0 │</pre>
<p>What if <em>last_name</em> would contain a comma? Let’s have a look at the raw input CSV for clarity sake:</p>
<!-- Ludwig van Beethoven enters the party -->
<!-- ```{console} -->
<!-- echo 'Ludwig,"Beethoven, van",1770,"Bonn,\nGermany"' >> composers.csv -->
<!-- ``` -->
<!-- bat -A composers.csv -->
<!-- //# cat composers.csv | csvquote | tr -d $'\x1f'   # comma (unit separator in unicode) -->
<!-- //# cat composers.csv | csvquote | tr $'\x1e' ' '  # new line (record separator in unicode) -->
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">names-comma.csv</span>
id,last_name,first_name,born
1,Williams,John,1932
2,Elfman,Danny,1953
3,Horner,James,1953
4,Shore,Howard,1946
5,Zimmer,Hans,1957
6,"Beethoven, van",Ludwig,1770</pre>
<p>Well, it appears that the first three approaches fail; all in different ways. Only <code>csvsql</code> is able to combine first_name and full_name:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names-comma.csv</span> <span style="color: #5f8700">sed</span> -re <span style="color: #af8700">'1s/.*/id,full_name,born/g;2,$s/(.*),(.*),(.*),(.*)/
\1,\3 \2,\4/g'</span> | <span style="color: #5f8700">tail</span> -n 1
6,"Beethoven,Ludwig  van",1770</pre>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names-comma.csv</span> <span style="color: #5f8700">awk</span> -F, <span style="color: #af8700">'BEGIN{OFS=","; print "id,full_name,born"} {if(NR &gt;
1) {print $1,$3" "$2,$4}}'</span> | <span style="color: #5f8700">tail</span> -n 1
6, van" "Beethoven,Ludwig</pre>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names-comma.csv</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">|</span> <span style="color: #5f8700">cols</span> -c first_name,last_name tr \",\" \" \" |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">header</span> -r full_name,id,born | <span style="color: #5f8700">csvcut</span> -c id,full_name,born | <span style="color: #5f8700">tail</span> -n 1
6,"Ludwig ""Beethoven  van""",1770</pre>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names-comma.csv</span> <span style="color: #5f8700">csvsql</span> --query <span style="color: #af8700">"SELECT id, first_name || ' ' || last_name AS
 full_name, born FROM stdin"</span> | <span style="color: #5f8700">tail</span> -n 1
6.0,"Ludwig Beethoven, van",1770.0</pre>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">names-comma.csv</span> <span style="color: #5f8700">rush</span> run -t <span style="color: #af8700">'unite(df, full_name, first_name, last_name, sep
 = " ")'</span> - | <span style="color: #5f8700">tail</span> -n 1
6,"Ludwig Beethoven, van",1770</pre>
<p>Wait a minute! What’s that last command? Is that R? Well, as a matter of fact, it is.
It’s R code evaluated through a command-line tool called <code>rush</code>. All that I can say at this moment, is that also this approach succeeds at merging the two columns.
I’ll discuss this nifty command-line tool later.</p>
</div>
<div id="combining-multiple-csv-files" class="section level3" number="5.4.6">
<h3 number="5.4.6"><span class="header-section-number">5.4.6</span> Combining Multiple CSV Files</h3>
<!-- #### Concatenate Vertically -->
<!-- Vertical concatenation may be necessary in cases where you have, for example, a data set which is generated on a daily basis, or where each data set represents a different, say, market or product. -->
<!-- Let’s simulate the former by splitting up *scientists.csv*, so that we have something to combine again. -->
<!-- <\!-- #TODO: use xsv split -\-> -->
<!-- <\!-- #TODO mention that this could also be useful for parallel processing (Chapter 8) -\-> -->
<!-- <\!-- #TODO: use xsv cat instead of csvstack -\-> -->
<!-- ```{console} -->
<!--  < scientists.xml xml2json | jq -r '.Root.data.record[] | [.field[0]."$t", .field[2]."$t", .field[3]."$t"] | @csv' | header -a "country,year,value" > scientists.csv -->
<!-- xsv split -s 2000 science scientists.csv -->
<!-- wc -l science/* -->
<!-- csvlook science/12000.csv -->
<!-- ``` -->
<!-- You could just concatenate the files using `cat` and removing the headers of all but the first file using as follows: -->
<!-- A more robust solution to combine multiple CSV files is to use `xsv`: -->
<!-- Out of order -->
<!-- ```{console} -->
<!-- xsv cat rows science/*.csv | tail -->
<!-- ``` -->
<!-- Note that the rows have -->
<!-- Back in order -->
<!-- ```{console} -->
<!-- echo $(ls -v science/*.csv) -->
<!-- xsv cat rows $(ls -v science/*.csv) | tail -->
<!-- ``` -->
<!-- ```{console} -->
<!-- awk 'NR==1 {H=$0; print} $0!=H' 2021-01-0*.csv | trim -->
<!-- ``` -->
<!-- ```{console} -->
<!-- shuf -ri 0-9999 -n 24 | nl -v0 -w1 -s, | header -a hour,value > 2021-01-01.csv -->
<!-- shuf -ri 0-9999 -n 24 | nl -v0 -w1 -s, | header -a hour,value > 2021-01-02.csv -->
<!-- shuf -ri 0-9999 -n 24 | nl -v0 -w1 -s, | header -a hour,value > 2021-01-03.csv -->
<!-- head -n 3 2021-*.csv -->
<!-- ``` -->
<!-- ```{console} -->
<!-- csvstack --group-name date --filenames 2021-*.csv | sed 's/\.csv,/,/' | csvlook -->
<!-- ``` -->
<!-- ```{console} -->
<!-- awk 'NR==1 {H=$0; print "date,"$0} $0!=H {print FILENAME","$0}' 2021-01-0*.csv | csvlook -->
<!-- ``` -->
<div id="concatenate-horizontally" class="section level4" number="5.4.6.1">
<h4 number="5.4.6.1"><span class="header-section-number">5.4.6.1</span> Concatenate Horizontally</h4>
<p>Let’s say you have three CSV files that want to put side by side. We use <code>tee</code><span class="citation"><a href="#fn81" class="footnote-ref" id="fnref81"><sup>81</sup></a></span> to save the result of <code>csvcut</code> in the middle of the pipeline:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #5f8700">csvcut</span> -c bill,tip | <span style="color: #5f8700">tee</span> bills.csv | <span style="color: #5f8700">head</span> -n 3 | <span style="color: #5f8700">csvlook</span>
│  bill │  tip │
├───────┼──────┤
│ 16.99 │ 1.01 │
│ 10.34 │ 1.66 │
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #5f8700">csvcut</span> -c day,time | <span style="color: #5f8700">tee</span> datetime.csv |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">head</span> -n 3 | <span style="color: #5f8700">csvlook</span> -I
│ day │ time   │
├─────┼────────┤
│ Sun │ Dinner │
│ Sun │ Dinner │
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #5f8700">csvcut</span> -c sex,smoker,size | <span style="color: #5f8700">tee</span> customers.csv |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">head</span> -n 3 | <span style="color: #5f8700">csvlook</span>
│ sex    │ smoker │ size │
├────────┼────────┼──────┤
│ Female │  False │    2 │
│ Male   │  False │    3 │</pre>
<p>Assuming that the rows line up, you can <code>paste</code><span class="citation"><a href="#fn82" class="footnote-ref" id="fnref82"><sup>82</sup></a></span> the files together:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">paste</span> -d, {bills,customers,datetime}.csv | <span style="color: #5f8700">head</span> -n 3 | <span style="color: #5f8700">csvlook</span> -I
│ bill  │ tip  │ sex    │ smoker │ size │ day │ time   │
├───────┼──────┼────────┼────────┼──────┼─────┼────────┤
│ 16.99 │ 1.01 │ Female │ No     │ 2    │ Sun │ Dinner │
│ 10.34 │ 1.66 │ Male   │ No     │ 3    │ Sun │ Dinner │</pre>
<p>Here, the command-line argument <code>-d</code> instructs <code>paste</code> to use a comma as the delimiter.</p>
</div>
<div id="joining" class="section level4" number="5.4.6.2">
<h4 number="5.4.6.2"><span class="header-section-number">5.4.6.2</span> Joining</h4>
<p>Sometimes data cannot be combined by vertical or horizontal concatenation.
In some cases, especially in relational databases, the data is spread over multiple tables (or files) in order to minimize redundancy.
Imagine you wanted to extend the Iris data set with more information about the three types of Iris flowers, namely the USDA identifier.
It so happens that I have a separate CSV file with these identifiers:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> <span style="text-decoration: underline">irismeta.csv</span>
│ species         │ wikipedia_url                                │ usda_id │
├─────────────────┼──────────────────────────────────────────────┼─────────┤
│ Iris-versicolor │ http://en.wikipedia.org/wiki/Iris_versicolor │ IRVE2   │
│ Iris-virginica  │ http://en.wikipedia.org/wiki/Iris_virginica  │ IRVI    │
│ Iris-setosa     │                                              │ IRSE    │</pre>
<p>What this data set and the Iris data set have in common is the <em>species</em> column.
You can use <code>csvjoin</code><span class="citation"><a href="#fn83" class="footnote-ref" id="fnref83"><sup>83</sup></a></span> to join the two data sets:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvjoin</span> -c species <span style="text-decoration: underline">iris.csv</span> <span style="text-decoration: underline">irismeta.csv</span> | <span style="color: #5f8700">csvcut</span> -c sepal_length,sepal_width,
species,usda_id | <span style="color: #5f8700">sed</span> -n <span style="color: #af8700">'1p;49,54p'</span> | <span style="color: #5f8700">csvlook</span>
│ sepal_length │ sepal_width │ species         │ usda_id │
├──────────────┼─────────────┼─────────────────┼─────────┤
│          4.6 │         3.2 │ Iris-setosa     │ IRSE    │
│          5.3 │         3.7 │ Iris-setosa     │ IRSE    │
│          5.0 │         3.3 │ Iris-setosa     │ IRSE    │
│          7.0 │         3.2 │ Iris-versicolor │ IRVE2   │
│          6.4 │         3.2 │ Iris-versicolor │ IRVE2   │
│          6.9 │         3.1 │ Iris-versicolor │ IRVE2   │</pre>
<p>Of course you can also use the SQL approach using <code>csvsql</code>, which is, as per usual, a bit longer (but potentially much more flexible):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvsql</span> --query <span style="color: #af8700">'SELECT i.sepal_length, i.sepal_width, i.species, m.usda_id FRO
M iris i JOIN irismeta m ON (i.species = m.species)'</span> <span style="text-decoration: underline">iris.csv</span> <span style="text-decoration: underline">irismeta.csv</span> | <span style="color: #5f8700">sed
</span> -n <span style="color: #af8700">'1p;49,54p'</span> | <span style="color: #5f8700">csvlook</span>
│ sepal_length │ sepal_width │ species         │ usda_id │
├──────────────┼─────────────┼─────────────────┼─────────┤
│          4.6 │         3.2 │ Iris-setosa     │ IRSE    │
│          5.3 │         3.7 │ Iris-setosa     │ IRSE    │
│          5.0 │         3.3 │ Iris-setosa     │ IRSE    │
│          7.0 │         3.2 │ Iris-versicolor │ IRVE2   │
│          6.4 │         3.2 │ Iris-versicolor │ IRVE2   │
│          6.9 │         3.1 │ Iris-versicolor │ IRVE2   │</pre>
<!-- ## JSON -->
<!-- ## XML -->
<!-- ## HTML -->
</div>
</div>
</div>
<div id="working-with-xmlhtml-and-json" class="section level2" number="5.5">
<h2 number="5.5"><span class="header-section-number">5.5</span> Working with XML/HTML and JSON</h2>
<p>In this section I’m going to demonstrate a couple of command-line tools that can convert data from one format to another.
There are two reasons to convert data.</p>
<p>First, oftentimes, the data needs to be in tabular form, just like a database table or a spreadsheet, because many visualization and machine learning algorithms depend on it.
CSV is inherently in tabular form, but JSON and HTML/XML data can have a deeply nested structure.</p>
<p>Second, many command-line tools, especially the classic ones such as <code>cut</code> and <code>grep</code>, operate on plain text.
This is because text is regarded as a universal interface between command-line tools.
Moreover, the other formats are younger. Each of these formats can be treated as plain text, allowing us to apply such command-line tools to the other formats as well.</p>
<p>Sometimes you can get away with applying the classic tools to structured data.
For example, by treating the JSON data below as plain text, you can change the attribute <em>gender</em> to <em>sex</em> using <code>sed</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">sed</span> -e <span style="color: #af8700">'s/"gender":/"sex":/g'</span> <span style="text-decoration: underline">users.json</span> | <span style="color: #5f8700">jq</span> | <span style="color: #5f8700">trim</span>
{
  "results": [
    {
      "sex": "male",
      "name": {
        "title": "mr",
        "first": "leevi",
        "last": "kivisto"
      },
      "location": {
… with 260 more lines</pre>
<p>Like many other command-line tools, <code>sed</code> does not make use of the structure of the data.
Better to either use a tool that makes use of the structure of the data (such as <code>jq</code> which I discuss below), or first convert the data to a tabular format such as CSV and then apply the appropriate command-line tool.</p>
<p>I’m going to demonstrate converting XML/HTML and JSON to CSV through a real-world use case.
The command-line tools that I’ll be using here are: <code>curl</code>, <code>pup</code><span class="citation"><a href="#fn84" class="footnote-ref" id="fnref84"><sup>84</sup></a></span>, <code>xml2json</code><span class="citation"><a href="#fn85" class="footnote-ref" id="fnref85"><sup>85</sup></a></span>, <code>jq</code> and <code>json2csv</code><span class="citation"><a href="#fn86" class="footnote-ref" id="fnref86"><sup>86</sup></a></span>.</p>
<p>Wikpedia holds a wealth of information. Much of this information is ordered in tables, which can be regarded as data sets.
For example, <a href="http://en.wikipedia.org/wiki/List_of_countries_and_territories_by_border/area_ratio">this page</a> contains a list of countries and territories together with their border length, their area, and the ratio between the two.</p>
<p>Let’s imagine that you’re interested in analyzing this data. In this section, I’ll walk you through all the necessary steps and their corresponding commands. I won’t go into every little detail, so it could be that you won’t understand everything right away. Don’t worry, I’re confident that you’ll get the gist of it. Remember that the purpose of this section is to demonstrate the command line. All tools and concepts used in this section (and more) will be explained in the subsequent chapters.</p>
<p>The data set that you’re interested in, is embedded in HTML.
Your goal is to end up with a representation of this data set that you can work with.
The very first step is to download the HTML using <code>curl</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">'http://en.wikipedia.org/wiki/List_of_countries_and_territories_by_bo
rder/area_ratio'</span> <span style="color: #af8700">&gt;</span> wiki.html</pre>
<p>The HTML is saved to a file named <em>wiki.html</em>.
Let’s see how the first 10 lines look like:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wiki.html</span> <span style="color: #5f8700">trim</span>
&lt;!DOCTYPE html&gt;
&lt;html class="client-nojs" lang="en" dir="ltr"&gt;
&lt;head&gt;
&lt;meta charset="UTF-8"/&gt;
&lt;title&gt;List of countries and territories by border/area ratio - Wikipedia&lt;/titl…
&lt;script&gt;document.documentElement.className="client-js";RLCONF={"wgBreakFrames":…
"Lists of countries by geography","Lists by area","Border-related lists"],"wgPa…
"wgGENewcomerTasksGuidanceEnabled":true,"wgGEAskQuestionEnabled":false,"wgGELin…
"ext.cx.eventlogging.campaigns","ext.centralNotice.geoIP","ext.centralNotice.st…
&lt;script&gt;(RLQ=window.RLQ||[]).push(function(){mw.loader.implement("user.options@…
… with 3070 more lines</pre>
<p>That seems to be in order.
Image that you’ve been able to determine that the root HTML element that we’re interested in is a <em>&lt;table&gt;</em> with the class <em>wikitable</em>.
This allows you to look at the part that you’re interest in using <code>grep</code> (the <code>-A</code> option specifies the number of lines you want to print after the matching line):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">grep</span> wikitable -A 21 <span style="text-decoration: underline">wiki.html</span>
&lt;table class="wikitable sortable"&gt;
&lt;tbody&gt;&lt;tr&gt;
&lt;th&gt;Rank&lt;/th&gt;
&lt;th&gt;Country or territory&lt;/th&gt;
&lt;th&gt;Total length of land borders (km)&lt;/th&gt;
&lt;th&gt;Total surface area (km&lt;sup&gt;2&lt;/sup&gt;)&lt;/th&gt;
&lt;th&gt;Border/area ratio (km/km&lt;sup&gt;2&lt;/sup&gt;)
&lt;/th&gt;&lt;/tr&gt;
&lt;tr&gt;
&lt;td&gt;1
&lt;/td&gt;
&lt;td&gt;Vatican City
&lt;/td&gt;
&lt;td&gt;3.2
&lt;/td&gt;
&lt;td&gt;0.44
&lt;/td&gt;
&lt;td&gt;7.2727273
&lt;/td&gt;&lt;/tr&gt;
&lt;tr&gt;
&lt;td&gt;2
&lt;/td&gt;</pre>
<p>You now actually see the countries and their values.
The next step is to extract the necessary elements from the HTML file.
For this you can use <code>pup</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wiki.html</span> <span style="color: #5f8700">pup</span> <span style="color: #af8700">'table.wikitable tbody'</span> | <span style="color: #5f8700">tee</span> table.html | <span style="color: #5f8700">trim</span>
&lt;tbody&gt;
 &lt;tr&gt;
  &lt;th&gt;
   Rank
  &lt;/th&gt;
  &lt;th&gt;
   Country or territory
  &lt;/th&gt;
  &lt;th&gt;
   Total length of land borders (km)
… with 4199 more lines</pre>
<p>The expression passed to <code>pup</code> is a CSS-selector.
The syntax is usually used to style web pages, but you can also use it to select certain elements from HTML.
In this case, you want to select the <em>tbody</em> of the <em>table</em> which has the <em>wikitable</em> class.
Up next is <code>xml2json</code>, which converts XML (and HTML) to JSON.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">table.html</span> <span style="color: #5f8700">xml2json</span> <span style="color: #af8700">&gt;</span> table.json
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">jq</span> <span style="text-decoration: underline">.</span> <span style="text-decoration: underline">table.json</span> | <span style="color: #5f8700">trim</span> 20
{
  "tbody": {
    "tr": [
      {
        "th": [
          {
            "$t": "Rank"
          },
          {
            "$t": "Country or territory"
          },
          {
            "$t": "Total length of land borders (km)"
          },
          {
            "$t": [
              "Total surface area (km",
              ")"
            ],
            "sup": {
… with 4691 more lines</pre>
<p>The reason I convert the HTML to JSON is because there is a very powerful tool called <code>jq</code> that operates on JSON data.
The following command extracts certain parts of the JSON data and reshapes it into a form that we can work with:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">table.json</span> <span style="color: #5f8700">jq</span> -r <span style="color: #af8700">'.tbody.tr[1:][] | [.td[]["$t"]] | @csv'</span> | <span style="color: #5f8700">header</span> -a rank,c
ountry,border,surface,ratio <span style="color: #af8700">&gt;</span> countries.csv</pre>
<p>The data is now in a form that you can work with.
Those were quite a few steps to get from a Wikipedia page to a CSV data set.
However, when you combine all of the above commands into one, you will see that it’s actually really concise and expressive.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> --max-column-width 28 <span style="text-decoration: underline">countries.csv</span>
│ rank │ country                      │    border │       surface │  ratio │
├──────┼──────────────────────────────┼───────────┼───────────────┼────────┤
│    1 │ Vatican City                 │      3.20 │          0.44 │ 7.273… │
│    2 │ Monaco                       │      4.40 │          2.00 │ 2.200… │
│    3 │ San Marino                   │     39.00 │         61.00 │ 0.639… │
│    4 │ Liechtenstein                │     76.00 │        160.00 │ 0.465… │
│    5 │ Sint Maarten  (Netherlands)  │     10.20 │         34.00 │ 0.300… │
│    6 │ Andorra                      │    120.30 │        468.00 │ 0.257… │
│    7 │ Gibraltar  (United Kingdom)  │      1.20 │          6.00 │ 0.200… │
│    8 │ Saint Martin (France)        │     10.20 │         54.00 │ 0.189… │
… with 238 more lines</pre>
<p>That concludes the demonstration of conversion XML/HTML to JSON to CSV.
While <code>jq</code> can perform many more operations, and while there exist specialized tools to work with XML data, in my experience, converting the data to CSV format as quickly as possible tends to work well.
This way you can spend more time becoming proficient at generic command-line tools, rather than very specific tools.</p>
</div>
<div id="summary-4" class="section level2" number="5.6">
<h2 number="5.6"><span class="header-section-number">5.6</span> Summary</h2>
<p>In this chapter we’ve looked at cleaning, or scrubbing, data.
As you’ve seen there is no single tool that can magically get rid of all the messiness of data; you’ll often need to combine multiple different tools to get the desired result.
Keep in mind that classic command-line tools such as <code>cut</code> and <code>sort</code> can’t interpret structured data.
Luckily, there are tools that convert one data format, such as JSON and XML, into another data format, such as CSV.
In the next chapter, which is again an intermezzo chapter, I’m going to show you how you can manage your project using <code>make</code>.
You’re free to skip this chapter if you can’t wait to start exploring and visualizing your data in <a href="#chapter-7-exploring-data">Chapter 7</a>.</p>
</div>
<div id="for-further-exploration-4" class="section level2" number="5.7">
<h2 number="5.7"><span class="header-section-number">5.7</span> For Further Exploration</h2>
<ul>
<li>I wish I could’ve explained more about <code>awk</code>. It’s such a powerful tool and programming language. I highly recommend that you take the time to learn it. Two good resources are the book <em>sed &amp; awk</em> by Doherty and Robbins and the online <a href="https://www.gnu.org/software/gawk/manual/gawk.html">GNU Awk User’s Guide</a>.</li>
<li>In this chapter I have used regular expressions in a couple of places. A tutorial about them is unfortunately beyond the scope of this book. Because regular expressions can be used in many different tools, I recommend that you learn about them. A good book is <em>Regular Expressions Cookbook</em> by Jan Goyvaerts and Steven Levithan.</li>
</ul>
<!--chapter:end:05.Rmd-->
</div>
</div>
<div id="chapter-6-project-management-with-make" class="section level1" number="6">
<h1 number="6"><span class="header-section-number">6</span> Project Management with Make</h1>
<p>I hope that by now you have come to appreciate that the command line is a very convenient environment for working with data.
You may have noticed that, as a consequence of working with the command line, we:</p>
<ul>
<li>Invoke many different commands.</li>
<li>Work from various directories.</li>
<li>Develop our own command-line tools.</li>
<li>Obtain and generate many (intermediate) files.</li>
</ul>
<p>Since this is an exploratory process, our workflow tends to be rather chaotic, which makes it difficult to keep track of what we’ve done.
It’s important that our steps can be reproduced, by us or by others.
When you continue with a project from some time ago, chances are that you have forgotten which commands you ran, from which directory, on which files, with which parameters, and in which order.
Imagine the challenges of sharing your project with a collaborator.</p>
<p>You can recover some commands by digging through the output of the <code>history</code> command, but this is, of course, not a reliable approach.
A somewhat better approach would be to save your commands to a shell script.
At least this allows you and your collaborators to reproduce the project.
A shell script is, however, also a sub-optimal approach because:</p>
<ul>
<li>It is difficult to read and to maintain.</li>
<li>Dependencies between steps are unclear.</li>
<li>Every step gets executed every time, which is inefficient and sometimes undesirable.</li>
</ul>
<p>This is where <code>make</code> really shines<span class="citation"><a href="#fn87" class="footnote-ref" id="fnref87"><sup>87</sup></a></span>. <code>make</code> is a command-line tool that allows you to:</p>
<ul>
<li>Formalize your data workflow steps in terms of input and output dependencies.</li>
<li>Run specific steps of your workflow.</li>
<li>Use inline code.</li>
<li>Store and retrieve data from external sources.</li>
</ul>

<div class="rmdnote">
In the first edition, this chapter used <code>drake</code><span class="citation"><a href="#fn88" class="footnote-ref" id="fnref88"><sup>88</sup></a></span> instead of <code>make</code>.
Drake was supposed to be a successor to <code>make</code> with additional features to work with data.
Unfortunately, Drake was abandoned by its creators in 2016 with too many unresolved bugs.
That’s why I’ve decided to use <code>make</code> instead.
</div>
<p>An important, related topic is <em>version control</em>, which allows you to track changes of your project, back up your project to a server, collaborate with others, and retrieve earlier versions when things go wrong.
A popular command-line tool to do version control is <code>git</code><span class="citation"><a href="#fn89" class="footnote-ref" id="fnref89"><sup>89</sup></a></span>.
It’s often used in combination with GitHub, an online service for distributed version control.
Many open source projects, including <a href="https://github.com/jeroenjanssens/data-science-at-the-command-line">this book</a>, are hosted on GitHub.
The topic of version control is beyond the scope of this book, but I highly recommend that you look into this, especially once you start collaborating with others.
At the end of this chapter I recommend a few resources to learn more.</p>
<div id="overview-3" class="section level2" number="6.1">
<h2 number="6.1"><span class="header-section-number">6.1</span> Overview</h2>
<p>Managing your data workflow with <code>make</code> is the main topic of this chapter.
As such, you’ll learn about:</p>
<ul>
<li>Defining your workflow with a <em>Makefile</em>.</li>
<li>Thinking about your workflow in terms of input and output dependencies.</li>
<li>Running tasks and building targets.</li>
</ul>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch06</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 28K
-rw-r--r-- 1 dst dst  37 Dec 14 11:49 Makefile.test
-rw-r--r-- 1 dst dst  16 Dec 14 11:49 numbers.make
-rw-r--r-- 1 dst dst  26 Dec 14 11:49 numbers-write.make
-rw-r--r-- 1 dst dst  21 Dec 14 11:49 numbers-write-var.make
-rw-r--r-- 1 dst dst 432 Dec 14 11:49 starwars.make
-rw-r--r-- 1 dst dst 263 Dec 14 11:49 tasks.make
-rw-r--r-- 1 dst dst  27 Dec 14 11:49 template.make</pre>
<p>The instructions to get these files are in <a href="#chapter-2-getting-started">Chapter 2</a>.
Any other files are either downloaded or generated using command-line tools.</p>
</div>
<div id="introducing-make" class="section level2" number="6.2">
<h2 number="6.2"><span class="header-section-number">6.2</span> Introducing Make</h2>
<p><code>make</code> organizes command execution around data and its dependencies.
Your data processing steps are formalized in a separate text file (a workflow).
Each step may have inputs and outputs.
<code>make</code> automatically resolves their dependencies and determines which commands need to be run and in which order.</p>
<p>This means that when you have, say, an SQL query that takes ten minutes, it only has to be executed when the result is missing or when the query has changed afterwards.
Also, if you want to (re-)run a specific step, <code>make</code> only re-runs the steps on which that step depends.
This can save you a lot of time.</p>
<p>Having a formalized workflow allows you to easily pick up your project after a few weeks and to collaborate with others. I strongly advise you to do this, even when you think this will be a one-off project, because you never know when you need to run certain steps again, or reuse them in another project.</p>
</div>
<div id="running-tasks" class="section level2" number="6.3">
<h2 number="6.3"><span class="header-section-number">6.3</span> Running Tasks</h2>
<p>By default, <code>make</code> searches for a configuration file called <em>Makefile</em> in the current directory.
It can also be named <em>makefile</em> (lower case), but I recommend calling your file <em>Makefile</em> because it’s more common and that way it appears at the top of a directory listing.
Normally you would only have one configuration file per project.
Because this chapter discusses many different ones, I haven given each of them a different filename with the <em>.make</em> extension.
Let’s start with the following <em>Makefile</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> -A <span style="text-decoration: underline">numbers.make</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">numbers.make</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ numbers:<span style="color: #af005f">␊</span>
   2   │ <span style="color: #00afaf">├──────┤</span>seq<span style="color: #00afaf">·</span>7<span style="color: #af005f">␊</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>This <em>Makefile</em> contains one <em>target</em> called <em><code>numbers</code></em>.
A <em>target</em> is like a task.
It’s usually the name of a file you’d like to create but it can also be more generic than that.
The line below, <em><code>seq 7</code></em>, is known as a <em>rule</em>.
Think of a rule as a recipe; one or more commands that specify how the target should be built.</p>
<p>The whitespace in front of the rule is a single tab character.
<code>make</code> is picky when it comes to whitespace.
Beware that some editors insert spaces when you press the <strong><code>TAB</code></strong> key, known as a soft tab, which will cause <code>make</code> to produce an error.
The following code illustrates this by expanding the tab to eight spaces:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">numbers.make</span> <span style="color: #5f8700">expand</span> <span style="color: #af8700">&gt;</span> spaces.make
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> -A <span style="text-decoration: underline">spaces.make
</span>───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">spaces.make</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ numbers:<span style="color: #af005f">␊</span>
   2   │ <span style="color: #00afaf">········</span>seq<span style="color: #00afaf">·</span>7<span style="color: #af005f">␊</span>
───────┴────────────────────────────────────────────────────────────────────────
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">make</span> -f <span style="text-decoration: underline">spaces.make</span> <span class="callout">&#10122;</span>
spaces.make:2: *** missing separator (did you mean TAB instead of 8 spaces?).  S
top. <span class="callout">&#10123;</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">rm</span> <span style="text-decoration: underline">spaces.make</span></pre>
<p><span class="callout">&#10122;</span> I need to add the <code>-f</code> option (short for the <code>--makefile</code> option) because the configuration file isn’t called <em>Makefile</em>, which is the default.
<br><span class="callout">&#10123;</span> One of the more helpful error messages you’ll find at the command line!</p>
<p>From now on, I’ll rename the appropriate file to <em>Makefile</em> because that matches real-world use more closely.
So, if I just run <code>make</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> <span style="text-decoration: underline">numbers.make</span> Makefile
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">make</span>
seq 7
1
2
3
4
5
6
7</pre>
<p>Then we see that <code>make</code> first prints the rule itself (<em><code>seq 7</code></em>), and then the output generated by the rule.
This process is known as <em>building</em> a target.
If you don’t specify the name of a target, then <code>make</code> will build the first target specified in the <em>Makefile</em>.
In practice though, you’ll most often be specifying the target you’d want to build:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">make</span> numbers
seq 7
1
2
3
4
5
6
7</pre>

<div class="rmdnote">
<code>make</code> was originally created to ease the compilation of source code, which explains some of the terminology like <em>target</em>, <em>rule</em>, and <em>building</em>.
</div>
<p>In this case, we’re not actually building anything, as in, we’re not creating any new files.
<code>make</code> will happily <em>build</em> our target <code>numbers</code> again, because it’s not finding a file called <em>numbers</em>.
In the next section I’ll go into this.</p>
<p>Sometimes it’s useful to have a target that builds regardless of whether a file with the same name exists.
Think of tasks that you need to perform as part of a project.
It’s good practice to declare those targets as phony by using a special target called <code>.PHONY</code> at the top of your <em>Makefile</em>, followed by the names of the phony targets.
Here’s an example <em>Makefile</em> that illustrates to use of phony targets:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">tasks.make</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">tasks.make</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #0087ff">.PHONY</span><span style="color: #af005f">:</span> <span style="color: #5f8700">clean publish docker-run</span>
   2   │
   3   │ <span style="color: #0087ff">clean</span><span style="color: #af005f">:</span>
   4   │         rm book/2e/book.md book/2e/render<span style="color: #af005f">*</span>.rds
   5   │
   6   │ <span style="color: #0087ff">publish</span><span style="color: #af005f">:</span>
   7   │         (<span style="color: #00afaf">cd</span> www <span style="color: #af005f">&amp;&amp;</span> hugo) <span style="color: #af005f">&amp;&amp;</span> netlify deploy --prod --dir www/public
   8   │
   9   │ <span style="color: #0087ff">docker-run</span><span style="color: #af005f">:
</span>  10   │         docker run -it --rm -v <span style="color: #af005f">$$(</span>pwd<span style="color: #af005f">)</span>/book/2e/data:/data -p 8000:8000
       │ datasciencetoolbox/dsatcl2e:latest <span class="callout">&#10122;</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p><span class="callout">&#10122;</span> Note the extra dollar sign in front of <em><code>$(pwd)</code></em>. This is needed because <code>make</code> uses a single dollar sign to refer to various special variables, which I’ll explain later.</p>
<p>The above is taken from a <em>Makefile</em> I use while working on this book.
You could say that I’m using <code>make</code> as a glorified task runner.
Although this wasn’t the primary purpose of <code>make</code>, it still provides a lot of value because I don’t need to remember or look up what incantation I used.
Instead, I type <code>make publish</code> and the latest version of the book is published.
It’s perfectly fine to put long-running commands in a <em>Makefile</em>.</p>
<p>And <code>make</code> can do much more for us!</p>
</div>
<div id="building-for-real" class="section level2" number="6.4">
<h2 number="6.4"><span class="header-section-number">6.4</span> Building, For Real</h2>
<p>Let’s modify our <em>Makefile</em> such the output of the rule is written to a file <em>numbers</em>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> <span style="text-decoration: underline">numbers-write.make</span> <span style="text-decoration: underline">Makefile</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">Makefile</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">Makefile</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #0087ff">numbers</span><span style="color: #af005f">:</span>
   2   │         seq 7 <span style="color: #af005f">&gt;</span> numbers
───────┴────────────────────────────────────────────────────────────────────────
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">make</span> numbers
seq 7 &gt; numbers
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">numbers</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">numbers</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ 1
   2   │ 2
   3   │ 3
   4   │ 4
   5   │ 5
   6   │ 6
   7   │ 7
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Now we can say that <code>make</code> is actually building something.
What’s more, if we run it again, we see that <code>make</code> reports that target <em><code>numbers</code></em> is up-to-date.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">make</span> <span style="text-decoration: underline">numbers</span>
make: 'numbers' is up to date.</pre>
<p>There’s no need to rebuild the target <em><code>numbers</code></em> because the file <em>numbers</em> already exists.
That’s great because <code>make</code> is saving us time by not repeating work.</p>
<p>In <code>make</code>, it’s all about files.
But keep in mind that <code>make</code> only cares about the <em>name</em> of the target.
It does not check whether a file of the same name actually gets created by the rule.
If we were to write to a file called <em>nummers</em>, which is Dutch for “numbers,” and the target was still called <em><code>numbers</code></em>, then <code>make</code> would always build this target. Vice versa, if the file <em>numbers</em> was created by some other process, whether automated or manual, then <code>make</code> would still consider that target up-to-date.</p>
<p>We can avoid some repetition by using the automatic variable <code>$@</code>, which gets expanded to the name of the target:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> <span style="text-decoration: underline">numbers-write-var.make</span> <span style="text-decoration: underline">Makefile</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">Makefile</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">Makefile</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #0087ff">numbers</span><span style="color: #af005f">:</span>
   2   │         seq 7 <span style="color: #af005f">&gt;</span> $@
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Let’s verify that this works by removing the file <em>numbers</em> and calling <code>make</code> again:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rm</span> <span style="text-decoration: underline">numbers</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">make</span> numbers
seq 7 &gt; numbers
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">numbers</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">numbers</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ 1
   2   │ 2
   3   │ 3
   4   │ 4
   5   │ 5
   6   │ 6
   7   │ 7
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Another reason for <code>make</code> to rebuild a target is its dependencies, so let’s discuss that next.</p>
</div>
<div id="adding-dependencies" class="section level2" number="6.5">
<h2 number="6.5"><span class="header-section-number">6.5</span> Adding Dependencies</h2>
<p>So far, we’ve looked at targets that exist in isolation.
In a typical data science workflow, many steps depend on other steps.
In order to properly talk about dependencies in a <em>Makefile</em>, let’s consider two tasks that work with a dataset about Star Wars characters.</p>
<p>Here’s an excerpt of that dataset:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">'https://raw.githubusercontent.com/tidyverse/dplyr/master/data-raw/st
arwars.csv'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">xsv</span> select name,height,mass,homeworld,species |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvlook</span>
│ name                  │ height │    mass │ homeworld      │ species        │
├───────────────────────┼────────┼─────────┼────────────────┼────────────────┤
│ Luke Skywalker        │    172 │    77.0 │ Tatooine       │ Human          │
│ C-3PO                 │    167 │    75.0 │ Tatooine       │ Droid          │
│ R2-D2                 │     96 │    32.0 │ Naboo          │ Droid          │
│ Darth Vader           │    202 │   136.0 │ Tatooine       │ Human          │
│ Leia Organa           │    150 │    49.0 │ Alderaan       │ Human          │
│ Owen Lars             │    178 │   120.0 │ Tatooine       │ Human          │
│ Beru Whitesun lars    │    165 │    75.0 │ Tatooine       │ Human          │
│ R5-D4                 │     97 │    32.0 │ Tatooine       │ Droid          │
… with 79 more lines</pre>
<p>The first task computes the ten tallest humans:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">'https://raw.githubusercontent.com/tidyverse/dplyr/master/data-raw/st
arwars.csv'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">grep</span> Human | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">cut</span> -d, -f 1,2 | <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> -t, -k2 -nr | <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">head</span> <span class="callout">&#10125;</span>
Darth Vader,202
Qui-Gon Jinn,193
Dooku,193
Bail Prestor Organa,191
Raymus Antilles,188
Mace Windu,188
Anakin Skywalker,188
Gregar Typho,185
Jango Fett,183
Cliegg Lars,183</pre>
<p><span class="callout">&#10122;</span> Only keep lines that contain the pattern <em><code>Human</code></em>.
<br><span class="callout">&#10123;</span> Extract the first two columns.
<br><span class="callout">&#10124;</span> Sort the lines by the second column in reverse numeric order.
<br><span class="callout">&#10125;</span> By default, <code>head</code> prints the first 10 lines. You can override this with the <code>-n</code> option.</p>
<p>The second task creates a box plot showing the distribution of heights per species (see Figure @ref(fig:starwars-image)):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -sL <span style="color: #af8700">'https://raw.githubusercontent.com/tidyverse/dplyr/master/data-raw/st
arwars.csv'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">rush</span> plot --x height --y species --geom boxplot <span style="color: #af8700">&gt;</span> heights.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">heights.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/heights.png" alt="Distribution of heights per species in Star Wars" width="90%" />
<p class="caption">
(#fig:starwars-image)Distribution of heights per species in Star Wars
</p>
</div>
<p>Let’s put these two tasks into a <em>Makefile</em>.
Instead of doing this incrementally, I’d first like to show what a complete <em>Makefile</em> looks like and then explain all the syntax step by step.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> <span style="text-decoration: underline">starwars.make</span> <span style="text-decoration: underline">Makefile</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">Makefile</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">Makefile</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ SHELL <span style="color: #af005f">:=</span> <span style="color: #5f8700">bash</span>
   2   │ <span style="color: #0087ff">.ONESHELL</span><span style="color: #af005f">:</span>
   3   │ .SHELLFLAGS <span style="color: #af005f">:=</span> <span style="color: #5f8700">-eu -o pipefail -c</span>
   4   │
   5   │ <span style="color: #0087ff">URL = "https</span><span style="color: #af005f">:</span><span style="color: #5f8700">//raw.githubusercontent.com/tidyverse/dplyr/master/data-ra
</span>       │ <span style="color: #5f8700">w/starwars.csv"</span>
   6   │
   7   │ <span style="color: #0087ff">.PHONY</span><span style="color: #af005f">:</span> <span style="color: #5f8700">all top10</span>
   8   │
   9   │ <span style="color: #0087ff">all</span><span style="color: #af005f">:</span> <span style="color: #5f8700">top10 heights.png</span>
  10   │
  11   │ <span style="color: #0087ff">data</span><span style="color: #af005f">:</span>
  12   │         mkdir $@
  13   │
  14   │ <span style="color: #0087ff">data/starwars.csv</span><span style="color: #af005f">:</span> <span style="color: #5f8700">data</span>
  15   │         curl -sL <span style="color: #af005f">$(</span>URL<span style="color: #af005f">)</span> <span style="color: #af005f">&gt;</span> $@
  16   │
  17   │ <span style="color: #0087ff">top10</span><span style="color: #af005f">:</span> <span style="color: #5f8700">data/starwars.csv</span>
  18   │         grep Human $&lt; <span style="color: #af005f">|</span>
  19   │         cut -d, -f 1,2 <span style="color: #af005f">|</span>
  20   │         sort -t, -k2 -nr <span style="color: #af005f">|</span>
  21   │         head
  22   │
  23   │ <span style="color: #0087ff">heights.png</span><span style="color: #af005f">:</span> <span style="color: #5f8700">data/starwars.csv</span>
  24   │         <span style="color: #af005f">&lt;</span> $&lt; rush plot --x height --y species --geom boxplot <span style="color: #af005f">&gt;</span> $@
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Let’s go through this <em>Makefile</em> step by step.
The first three lines are there to change some default settings related to <code>make</code> itself:</p>
<ol style="list-style-type: decimal">
<li>All rules are executed in a shell, which by default, is <code>sh</code>. With the <em><code>SHELL</code></em> variable we can change this to another shell, like <code>bash</code>. This way we can use everything that Bash has to offer such as for loops.</li>
<li>By default, every line in a rule is sent separately to the shell. With the special target <em><code>.ONESHELL</code></em> we can override this so the rule for target <em><code>top10</code></em> works.</li>
<li>The <em><code>.SHELLFLAGS</code></em> line makes Bash more strict, which is considered a <a href="http://redsymbol.net/articles/unofficial-bash-strict-mode/">best practice</a>. For example, because of this, the pipeline in the rule for target <em><code>top10</code></em> now stops as soon as there is an error.</li>
</ol>
<p>We define a custom variable called <em><code>URL</code></em>.
Even though this is only used once, I find it helpful to put information like this near the beginning of the file, so that you can easily make changes to these kinds of settings.</p>
<p>With the special target <em><code>.PHONY</code></em> we can indicate which targets are not represented by files. In our case that holds for targets <em><code>all</code></em> and <em><code>top10</code></em>. These targets will now be executed regardless of whether the directory contains files with the same name.</p>
<p>There are five targets: <em><code>all</code></em>, <em><code>data</code></em>, <em><code>data/starwars.csv</code></em>, <em><code>top10</code></em>, and <em><code>heights.png</code></em>.
Figure @ref(fig:starwars-image) provides an overview of these targets and the dependencies between them.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0602.png" alt="Dependencies between targets" width="90%" />
<p class="caption">
(#fig:dependencies)Dependencies between targets
</p>
</div>
<p>Let’s discuss each target in turn:</p>
<ol style="list-style-type: decimal">
<li>The target <em><code>all</code></em> has two dependencies but no rule. This is like a shortcut to execute one or more targets in the order in which they are specified. In this case: <em><code>top10</code></em> and <em><code>heights.png</code></em>. The target <em><code>all</code></em> appears as the first target in the <em>Makefile</em>, which means that if we run <code>make</code>, this target will be built.</li>
<li>The target <em><code>data</code></em> creates the directory <em>data</em>. Earlier I said that <code>make</code> is all about files. Well, it’s also about directories. This target will only be executed when the directory <em>data</em> doesn’t yet exist.</li>
<li>The target <em><code>data/starwars.csv</code></em> depends on the target <em><code>data</code></em>. If there’s no <em><code>data</code></em> directory, it will first be created. Once all dependencies are satisfied, the rule will be executed, which involves downloading a file and saving it to a file with the same name as the target.</li>
<li>The target <em><code>top10</code></em> is marked as phony, so it will always be built if specified. It depends on the <em><code>data/starwars.csv</code></em> target. It makes use of a special variable, <em><code>$&lt;</code></em> which expands to the name of the first prerequisite, namely <em>data/starwars.csv</em>.</li>
<li>The target <em><code>heights.png</code></em>, like target <em><code>top10</code></em> depends <em><code>data/starwars.csv</code></em> and makes use of both automatic variables we’ve seen in this chapter. See the <a href="https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html">online documentation</a> if you’d like to learn about other automatic variables.</li>
</ol>
<p>Last but not least, let’s verify that this <em>Makefile</em> works:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">make</span>
mkdir data
curl -sL "https://raw.githubusercontent.com/tidyverse/dplyr/master/data-raw/star
wars.csv" &gt; data/starwars.csv
grep Human data/starwars.csv |
cut -d, -f 1,2 |
sort -t, -k2 -nr |
head
Darth Vader,202
Qui-Gon Jinn,193
Dooku,193
Bail Prestor Organa,191
Raymus Antilles,188
Mace Windu,188
Anakin Skywalker,188
Gregar Typho,185
Jango Fett,183
Cliegg Lars,183
&lt; data/starwars.csv rush plot --x height --y species --geom boxplot &gt; heights.pn
g</pre>
<p>No surprises here. Because we didn’t specify any target, the <em><code>all</code></em> target will be built, which, in turn, causes both the <em><code>top10</code></em> and <em><code>heights.png</code></em> targets to be built. The output of the former is printed to standard output and the latter creates a file <em>heights.png</em>. The <em>data</em> directory is created only once, just like the CSV file is only downloaded once.</p>
<p>There’s nothing more fun than just playing with your data and forgetting everything else.
But you have to trust me when I say that it’s worthwhile to keep a record of what you have done using a <em>Makefile</em>.
Not only will it make your life easier (pun intended), but you will also start thinking about your data workflow in terms of steps.
Just as with your own command-line toolbox, which you expand over time, the same holds for <code>make</code> workflows.
The more steps you have defined, the easier it gets to keep doing it, because very often you can reuse certain steps.
I hope that you will get used to <code>make</code>, and that it will make your life easier.</p>
</div>
<div id="summary-5" class="section level2" number="6.6">
<h2 number="6.6"><span class="header-section-number">6.6</span> Summary</h2>
<p>One of the beauties of the command line is that it allows you to play with your data.
You can easily execute different commands and process different data files.
It is a very interactive and iterative process.
After a while, it is easy to forget which steps you have taken to get the desired result.
It’s therefore very important to document your steps every once in a while.
This way, if you or one of your colleagues picks up your project after some time, the same result can be produced again by executing the same steps.</p>
<p>In this chapter I’ve shown you that just putting every command in one Bash script is suboptimal.
Instead, I proposed to use <code>make</code> as a command-line tool to manage your data workflow.
The next chapter covers the third step of the OSEMN model for data science namely exploring data.</p>
</div>
<div id="for-further-exploration-5" class="section level2" number="6.7">
<h2 number="6.7"><span class="header-section-number">6.7</span> For Further Exploration</h2>
<ul>
<li>The book <em>Managing Projects with GNU Make</em> by Robert Mecklenburg and the online <em>GNU Make Manual</em> provide a comprehensive and advanced overview of <code>make</code>.</li>
<li>There exist plenty of other workflow managers besides <code>make</code>. Although they differ in syntax and features, they also use concepts such as targets, rules, and dependencies. Examples include <a href="https://luigi.readthedocs.io">Luigi</a>, <a href="https://airflow.apache.org">Apache Airflow</a>, and <a href="https://www.nextflow.io">Nextflow</a>.</li>
<li>To learn more about version control, in particular <code>git</code> and GitHub, I recommend the book <em>Pro Git</em> by Scott Chacon and Ben Straub. It’s <a href="https://git-scm.com/book/en/v2">available for free</a>. The <a href="https://docs.github.com/en/get-started">online GitHub documentation</a> is also a great starting point.</li>
</ul>
<!--chapter:end:06.Rmd-->
</div>
</div>
<div id="chapter-7-exploring-data" class="section level1" number="7">
<h1 number="7"><span class="header-section-number">7</span> Exploring Data</h1>
<p>After all that hard work (unless you already had clean data lying around), it’s time for some fun.
Now that you have obtained and scrubbed your data, you can continue with the third step of the OSEMN model, which is to explore it.</p>
<p>Exploring is the step where you familiarize yourself with the data.
Being familiar with the data is essential when you want to extract any value from it.
For example, knowing what kind of features the data has, means you know which ones are worth further exploration and which ones you can use to answer any questions that you have.</p>
<p>Exploring your data can be done from three perspectives.
The first perspective is to inspect the data and its properties.
Here, you want to find out things like what the raw data looks like, how many data points the dataset has, and which features the dataset has.</p>
<p>The second is to compute descriptive statistics. This perspective is useful for learning more about the individual features.
The output is often brief and textual and can therefore be printed on the command line.</p>
<p>The third perspective is to create visualizations of the data. From this perspective you can gain insight into how multiple features interact. I’ll discuss a way of creating visualizations that can be printed on the command line. However, visualizations are best suited to be displayed on a graphical user interface. An advantage of data visualizations over descriptive statistics is that they are more flexible and that they can convey much more information.</p>
<div id="overview-4" class="section level2" number="7.1">
<h2 number="7.1"><span class="header-section-number">7.1</span> Overview</h2>
<p>In this chapter, you’ll learn how to:</p>
<ul>
<li>Inspect the data and its properties</li>
<li>Compute descriptive statistics</li>
<li>Create data visualizations inside and outside the command line</li>
</ul>
<p>This chapter starts with the following files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch07</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 104K
-rw-r--r-- 1 dst dst  125 Dec 14 11:49 datatypes.csv
-rw-r--r-- 1 dst dst 7.8K Dec 14 11:49 tips.csv
-rw-r--r-- 1 dst dst  83K Dec 14 11:49 venture.csv
-rw-r--r-- 1 dst dst 4.6K Dec 14 11:49 venture-wide.csv</pre>
<p>The instructions to get these files are in <a href="#chapter-2-getting-started">Chapter 2</a>.
Any other files are either downloaded or generated using command-line tools.</p>
</div>
<div id="inspecting-data-and-its-properties" class="section level2" number="7.2">
<h2 number="7.2"><span class="header-section-number">7.2</span> Inspecting Data and its Properties</h2>
<p>In this section I’ll demonstrate how to inspect your dataset and its properties. Because the upcoming visualization and modeling techniques expect the data to be in a rectangular shape, I’ll assume that the data is in CSV format. You can use the techniques described in <a href="#chapter-5-scrubbing-data">Chapter 5</a> to convert your data to CSV if necessary.</p>
<p>For simplicity sake, I’ll also assume that your data has a header.
In the first subsection I’ll show a way to determine whether that’s the case.
Once you know you have a header, you can continue answering the following questions:</p>
<ul>
<li>How many data points and features does the dataset have?</li>
<li>What does the raw data look like?</li>
<li>What kind of features does the dataset have?</li>
<li>Can some of these features be treated as categorical?</li>
</ul>
<div id="header-or-not-here-i-come" class="section level3" number="7.2.1">
<h3 number="7.2.1"><span class="header-section-number">7.2.1</span> Header Or Not, Here I Come</h3>
<p>You can check whether your file has a header by printing the first few lines using <code>head</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">head</span> -n 5 <span style="text-decoration: underline">venture.csv</span>
FREQ,TIME_FORMAT,TIME_PERIOD,EXPEND,UNIT,GEO,OBS_STATUS,OBS_VALUE,FREQ_DESC,TIME
_FORMAT_DESC,TIME_PERIOD_DESC,OBS_STATUS_DESC,EXPEND_DESC,UNIT_DESC,GEO_DESC
A,P1Y,2015,INV_VEN,PC_GDP,CZ,,0.002,Annual,Annual,Year 2015,No data,"Venture cap
ital investment (seed, start-up and later stage) ",Percentage of GDP,Czechia
A,P1Y,2007,INV_VEN,PC_GDP,DE,,0.034,Annual,Annual,Year 2007,No data,"Venture cap
ital investment (seed, start-up and later stage) ",Percentage of GDP,Germany
A,P1Y,2008,INV_VEN,PC_GDP,DE,,0.039,Annual,Annual,Year 2008,No data,"Venture cap
ital investment (seed, start-up and later stage) ",Percentage of GDP,Germany
A,P1Y,2009,INV_VEN,PC_GDP,DE,,0.029,Annual,Annual,Year 2009,No data,"Venture cap
ital investment (seed, start-up and later stage) ",Percentage of GDP,Germany</pre>
<p>If the lines wrap around, add line numbers using <code>nl</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">head</span> -n 3 <span style="text-decoration: underline">venture.csv</span> | <span style="color: #5f8700">nl</span>
     1  FREQ,TIME_FORMAT,TIME_PERIOD,EXPEND,UNIT,GEO,OBS_STATUS,OBS_VALUE,FREQ_D
ESC,TIME_FORMAT_DESC,TIME_PERIOD_DESC,OBS_STATUS_DESC,EXPEND_DESC,UNIT_DESC,GEO_
DESC
     2  A,P1Y,2015,INV_VEN,PC_GDP,CZ,,0.002,Annual,Annual,Year 2015,No data,"Ven
ture capital investment (seed, start-up and later stage) ",Percentage of GDP,Cze
chia
     3  A,P1Y,2007,INV_VEN,PC_GDP,DE,,0.034,Annual,Annual,Year 2007,No data,"Ven
ture capital investment (seed, start-up and later stage) ",Percentage of GDP,Ger
many</pre>
<p>Alternatively, you can use <code>trim</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">venture.csv</span> <span style="color: #5f8700">trim</span> 5
FREQ,TIME_FORMAT,TIME_PERIOD,EXPEND,UNIT,GEO,OBS_STATUS,OBS_VALUE,FREQ_DESC,TIM…
A,P1Y,2015,INV_VEN,PC_GDP,CZ,,0.002,Annual,Annual,Year 2015,No data,"Venture ca…
A,P1Y,2007,INV_VEN,PC_GDP,DE,,0.034,Annual,Annual,Year 2007,No data,"Venture ca…
A,P1Y,2008,INV_VEN,PC_GDP,DE,,0.039,Annual,Annual,Year 2008,No data,"Venture ca…
A,P1Y,2009,INV_VEN,PC_GDP,DE,,0.029,Annual,Annual,Year 2009,No data,"Venture ca…
… with 536 more lines</pre>
<p>In this case, it’s clear that the first line is a header because it contains only uppercase names and the subsequent lines contain numbers.
This is indeed quite a subjective process and it’s up to you to decide whether the first line is a header or already the first data point.
When the dataset contains no header, you’re best off using the <code>header</code> tool (discussed in <a href="#chapter-5-scrubbing-data">Chapter 5</a>) to correct that.</p>
</div>
<div id="inspect-all-the-data" class="section level3" number="7.2.2">
<h3 number="7.2.2"><span class="header-section-number">7.2.2</span> Inspect All The Data</h3>
<p>If you want to inspect the raw data at your own pace, then it’s probably not a good idea to use <code>cat</code>, because then all the data will be printed in one go.
I recommend using <code>less</code><span class="citation"><a href="#fn90" class="footnote-ref" id="fnref90"><sup>90</sup></a></span>, which allows you to interactively inspect your data in the command line.
You can prevent long lines (as with <em>venture.csv</em>) from wrapping by specifying the <code>-S</code> option:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">less</span> -S <span style="text-decoration: underline">venture.csv</span></pre>
<pre>FREQ,TIME_FORMAT,TIME_PERIOD,EXPEND,UNIT,GEO,OBS_STATUS,OBS_VALUE,FREQ_DESC,TIM<span style="font-style: italic">&gt;
</span>A,P1Y,2015,INV_VEN,PC_GDP,CZ,,0.002,Annual,Annual,Year 2015,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2007,INV_VEN,PC_GDP,DE,,0.034,Annual,Annual,Year 2007,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2008,INV_VEN,PC_GDP,DE,,0.039,Annual,Annual,Year 2008,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2009,INV_VEN,PC_GDP,DE,,0.029,Annual,Annual,Year 2009,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2010,INV_VEN,PC_GDP,DE,,0.029,Annual,Annual,Year 2010,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2011,INV_VEN,PC_GDP,DE,,0.029,Annual,Annual,Year 2011,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2012,INV_VEN,PC_GDP,DE,,0.021,Annual,Annual,Year 2012,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2013,INV_VEN,PC_GDP,DE,,0.023,Annual,Annual,Year 2013,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2014,INV_VEN,PC_GDP,DE,,0.021,Annual,Annual,Year 2014,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2015,INV_VEN,PC_GDP,DE,,0.025,Annual,Annual,Year 2015,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2007,INV_VEN,PC_GDP,DK,,0.092,Annual,Annual,Year 2007,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2008,INV_VEN,PC_GDP,DK,,0.074,Annual,Annual,Year 2008,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2009,INV_VEN,PC_GDP,DK,,0.051,Annual,Annual,Year 2009,No data,"Venture ca<span style="font-style: italic">&gt;
</span>A,P1Y,2010,INV_VEN,PC_GDP,DK,,0.059,Annual,Annual,Year 2010,No data,"Venture ca<span style="font-style: italic">&gt;
</span>:                   </pre>
<p>The greater-than signs on the right indicate that you can scroll horizontally.
You can scroll up and down by pressing <strong><code>Up</code></strong> and <strong><code>Down</code></strong>.
Press <strong><code>Space</code></strong> to scroll down an entire screen.
Scrolling horizontally is done by pressing <strong><code>Left</code></strong> and <strong><code>Right</code></strong>.
Press <strong><code>g</code></strong> and <strong><code>G</code></strong> to go to start and the end of the file, respectively.
Quitting <code>less</code> is done by pressing <strong><code>q</code></strong>.
The manual page lists all the available key bindings.</p>
<p>One advantage of <code>less</code> is that it does not load the entire file into memory, which means it’s fast even for viewing large files.</p>
</div>
<div id="feature-names-and-data-types" class="section level3" number="7.2.3">
<h3 number="7.2.3"><span class="header-section-number">7.2.3</span> Feature Names and Data Types</h3>
<p>The column (or feature) names may indicate the meaning of the feature.
You can use the following <code>head</code> and <code>tr</code> combo for this:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">venture.csv</span> <span style="color: #5f8700">head</span> -n 1 | <span style="color: #5f8700">tr</span> , <span style="color: #af8700">'\n'</span>
FREQ
TIME_FORMAT
TIME_PERIOD
EXPEND
UNIT
GEO
OBS_STATUS
OBS_VALUE
FREQ_DESC
TIME_FORMAT_DESC
TIME_PERIOD_DESC
OBS_STATUS_DESC
EXPEND_DESC
UNIT_DESC
GEO_DESC</pre>
<p>This basic command assumes that the file is delimited by commas.
A more robust approach is to use <code>csvcut</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvcut</span> -n <span style="text-decoration: underline">venture.csv</span>
  1: FREQ
  2: TIME_FORMAT
  3: TIME_PERIOD
  4: EXPEND
  5: UNIT
  6: GEO
  7: OBS_STATUS
  8: OBS_VALUE
  9: FREQ_DESC
 10: TIME_FORMAT_DESC
 11: TIME_PERIOD_DESC
 12: OBS_STATUS_DESC
 13: EXPEND_DESC
 14: UNIT_DESC
 15: GEO_DESC</pre>
<p>You can go a step further than just printing the column names.
Besides the names of the columns, it would be very useful to know what type of values each column contains, such as a string of characters, a numerical value, or a date.
Assume that you have the following toy dataset:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> -A <span style="text-decoration: underline">datatypes.csv</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">datatypes.csv</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ a,b,c,d,e,f<span style="color: #af005f">␊</span>
   2   │ 1,0.0,FALSE,"""Yes!""",2011-11-11<span style="color: #00afaf">·</span>11:00,2012-09-08<span style="color: #af005f">␊</span>
   3   │ 42,3.1415,TRUE,"OK,<span style="color: #00afaf">·</span>good",2014-09-15,12/6/70<span style="color: #af005f">␊</span>
   4   │ 66,,False,2198,,<span style="color: #af005f">␊</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Which <code>csvlook</code> interprets as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> <span style="text-decoration: underline">datatypes.csv</span>
│  a │      b │     c │ d        │                   e │          f │
├────┼────────┼───────┼──────────┼─────────────────────┼────────────┤
│  1 │ 0.000… │ False │ "Yes!"   │ 2011-11-11 11:00:00 │ 2012-09-08 │
│ 42 │ 3.142… │  True │ OK, good │ 2014-09-15 00:00:00 │ 1970-12-06 │
│ 66 │        │ False │ 2198     │                     │            │</pre>
<p>I have already used <code>csvsql</code> in <a href="#chapter-5-scrubbing-data">Chapter 5</a> to execute SQL queries directly on CSV data.
When no command-line arguments are passed, it generates the necessary SQL statement that would be needed if you were to insert this data into an actual database.
You can use the output also to inspect what the inferred column types are.
If a column has the <em>NOT NULL</em> string printed after the data type, then that column contains no missing values.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvsql</span> <span style="text-decoration: underline">datatypes.csv</span>
CREATE TABLE datatypes (
        a DECIMAL NOT NULL,
        b DECIMAL,
        c BOOLEAN NOT NULL,
        d VARCHAR NOT NULL,
        e TIMESTAMP,
        f DATE
);</pre>
<p>This output is especially useful when you use other tools within the <code>csvkit</code> suite, such as <code>csvgrep</code>, <code>csvsort</code> and <code>csvsql</code>.
For <em>venture.csv</em>, the columns are inferred as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvsql</span> <span style="text-decoration: underline">venture.csv
</span>CREATE TABLE venture (
        "FREQ" VARCHAR NOT NULL,
        "TIME_FORMAT" VARCHAR NOT NULL,
        "TIME_PERIOD" DECIMAL NOT NULL,
        "EXPEND" VARCHAR NOT NULL,
        "UNIT" VARCHAR NOT NULL,
        "GEO" VARCHAR NOT NULL,
        "OBS_STATUS" BOOLEAN,
        "OBS_VALUE" DECIMAL NOT NULL,
        "FREQ_DESC" VARCHAR NOT NULL,
        "TIME_FORMAT_DESC" VARCHAR NOT NULL,
        "TIME_PERIOD_DESC" VARCHAR NOT NULL,
        "OBS_STATUS_DESC" VARCHAR NOT NULL,
        "EXPEND_DESC" VARCHAR NOT NULL,
        "UNIT_DESC" VARCHAR NOT NULL,
        "GEO_DESC" VARCHAR NOT NULL
);</pre>
</div>
<div id="unique-identifiers-continuous-variables-and-factors" class="section level3" number="7.2.4">
<h3 number="7.2.4"><span class="header-section-number">7.2.4</span> Unique Identifiers, Continuous Variables, and Factors</h3>
<p>Knowing the data type of each feature is not enough.
It’s also essential to know what each feature represents.
Having knowledge about the domain is very useful here, but we may also get some context by looking at the data itself.</p>
<p>Both a string and an integer could be a unique identifier or could represent a category.
In the latter case, this could be used to assign a color to your visualization.
But if an integer denotes, say, a postal code, then it doesn’t make sense to compute the average.</p>
<p>To determine whether a feature should be treated as a unique identifier or categorical variable, you could count the number of unique values for a specific column:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">wc</span> -l <span style="text-decoration: underline">tips.csv</span>
245 tips.csv
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #5f8700">csvcut</span> -c day | <span style="color: #5f8700">header</span> -d | <span style="color: #5f8700">sort</span> | <span style="color: #5f8700">uniq</span> | <span style="color: #5f8700">wc</span> -l
4</pre>
<p>You can use <code>csvstat</code><span class="citation"><a href="#fn91" class="footnote-ref" id="fnref91"><sup>91</sup></a></span>, which is part of <code>csvkit</code>, to get the number of unique values for each column:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstat</span> <span style="text-decoration: underline">tips.csv</span> --unique
  1. bill: 229
  2. tip: 123
  3. sex: 2
  4. smoker: 2
  5. day: 4
  6. time: 2
  7. size: 6
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstat</span> <span style="text-decoration: underline">venture.csv</span> --unique
  1. FREQ: 1
  2. TIME_FORMAT: 1
  3. TIME_PERIOD: 9
  4. EXPEND: 1
  5. UNIT: 3
  6. GEO: 20
  7. OBS_STATUS: 1
  8. OBS_VALUE: 286
  9. FREQ_DESC: 1
 10. TIME_FORMAT_DESC: 1
 11. TIME_PERIOD_DESC: 9
 12. OBS_STATUS_DESC: 1
 13. EXPEND_DESC: 1
 14. UNIT_DESC: 3
 15. GEO_DESC: 20</pre>
<p>If there’s only one unique value (such as with <em>OBS_STATUS</em>), then there’s a chance that you can discard that column because it doesn’t provide any value.
If you wanted to automatically discard all such columns, then you could use the following pipeline:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">venture.csv</span> <span style="color: #5f8700">csvcut</span> -C <span style="color: #af005f">$(</span> <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span>   <span style="color: #5f8700">csvstat</span> <span style="text-decoration: underline">venture.csv</span> --unique | <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span>   <span style="color: #5f8700">grep</span> <span style="color: #af8700">': 1$'</span> | <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span>   <span style="color: #5f8700">cut</span> -d. -f 1 | <span class="callout">&#10125;</span>
<span style="font-weight: bold">&gt;</span>   <span style="color: #5f8700">tr</span> -d <span style="color: #af8700">' '</span> | <span class="callout">&#10126;</span>
<span style="font-weight: bold">&gt;</span>   <span style="color: #5f8700">paste</span> -sd, <span class="callout">&#10127;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af005f">)</span> | <span style="color: #5f8700">trim</span> <span class="callout">&#10128;</span>
TIME_PERIOD,UNIT,GEO,OBS_VALUE,TIME_PERIOD_DESC,UNIT_DESC,GEO_DESC
2015,PC_GDP,CZ,0.002,Year 2015,Percentage of GDP,Czechia
2007,PC_GDP,DE,0.034,Year 2007,Percentage of GDP,Germany
2008,PC_GDP,DE,0.039,Year 2008,Percentage of GDP,Germany
2009,PC_GDP,DE,0.029,Year 2009,Percentage of GDP,Germany
2010,PC_GDP,DE,0.029,Year 2010,Percentage of GDP,Germany
2011,PC_GDP,DE,0.029,Year 2011,Percentage of GDP,Germany
2012,PC_GDP,DE,0.021,Year 2012,Percentage of GDP,Germany
2013,PC_GDP,DE,0.023,Year 2013,Percentage of GDP,Germany
2014,PC_GDP,DE,0.021,Year 2014,Percentage of GDP,Germany
… with 531 more lines</pre>
<p><span class="callout">&#10122;</span> The <code>-C</code> option deselects columns given their locations (or names), which is provided with command substitution
<br><span class="callout">&#10123;</span> Obtain the number of unique values for each column in <em>venture.csv</em>
<br><span class="callout">&#10124;</span> Only keep the columns that contain one unique value
<br><span class="callout">&#10125;</span> Extract the column location
<br><span class="callout">&#10126;</span> Trim any white space
<br><span class="callout">&#10127;</span> Put all column locations on one comma-separated line
<br><span class="callout">&#10128;</span> Only show the first 10 lines</p>
<p>Having said that, I’m going to keep those columns for now.</p>
<p>Generally speaking, if the number of unique values is low compared to the total number of rows, then that feature might be treated as a categorical one (such as <em>GEO</em> in the case of <em>venture.csv</em>).
If the number is equal to the number of rows, it might be a unique identifier but it might also be a numerical value.
There’s only one way to find out: we need to go deeper.</p>
</div>
</div>
<div id="computing-descriptive-statistics" class="section level2" number="7.3">
<h2 number="7.3"><span class="header-section-number">7.3</span> Computing Descriptive Statistics</h2>
<div id="column-statistics" class="section level3" number="7.3.1">
<h3 number="7.3.1"><span class="header-section-number">7.3.1</span> Column Statistics</h3>
<p>The command-line tool <code>csvstat</code> gives a lot of information. For each feature (column), it shows:</p>
<ul>
<li>The data type</li>
<li>Whether it has any missing values (nulls)</li>
<li>The number of unique values</li>
<li>Various descriptive statistics (minimum, maximum, sum, mean, standard deviation, and median) for those features for which it is appropriate</li>
</ul>
<p>Invoke <code>csvstat</code> as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstat</span> <span style="text-decoration: underline">venture.csv</span> | <span style="color: #5f8700">trim</span> 32
  1. "FREQ"
 
        Type of data:          Text
        Contains null values:  False
        Unique values:         1
        Longest value:         1 characters
        Most common values:    A (540x)
 
  2. "TIME_FORMAT"
 
        Type of data:          Text
        Contains null values:  False
        Unique values:         1
        Longest value:         3 characters
        Most common values:    P1Y (540x)
 
  3. "TIME_PERIOD"
 
        Type of data:          Number
        Contains null values:  False
        Unique values:         9
        Smallest value:        2,007
        Largest value:         2,015
        Sum:                   1,085,940
        Mean:                  2,011
        Median:                2,011
        StDev:                 2.584
        Most common values:    2,015 (60x)
                               2,007 (60x)
                               2,008 (60x)
                               2,009 (60x)
                               2,010 (60x)
… with 122 more lines</pre>
<p>I’m only showing the first 32 lines, because this produces a lot of output. You might want to pipe this through <code>less</code>.
If you’re only interested in a specific statistic, you can also use one of the following options:</p>
<ul>
<li><code>--max</code> (maximum)</li>
<li><code>--min</code> (minimum)</li>
<li><code>--sum</code> (sum)</li>
<li><code>--mean</code> (mean)</li>
<li><code>--median</code> (median)</li>
<li><code>--stdev</code> (standard deviation)</li>
<li><code>--nulls</code> (whether column contains nulls)</li>
<li><code>--unique</code> (unique values)</li>
<li><code>--freq</code> (frequent values)</li>
<li><code>--len</code> (maximum value length)</li>
</ul>
<p>For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstat</span> <span style="text-decoration: underline">venture.csv</span> --freq | <span style="color: #5f8700">trim</span>
  1. FREQ: { "A": 540 }
  2. TIME_FORMAT: { "P1Y": 540 }
  3. TIME_PERIOD: { "2015": 60, "2007": 60, "2008": 60, "2009": 60, "2010": 60 }
  4. EXPEND: { "INV_VEN": 540 }
  5. UNIT: { "PC_GDP": 180, "NR_COMP": 180, "MIO_EUR": 180 }
  6. GEO: { "CZ": 27, "DE": 27, "DK": 27, "EL": 27, "ES": 27 }
  7. OBS_STATUS: { "None": 540 }
  8. OBS_VALUE: { "0": 28, "1": 19, "2": 14, "0.002": 10, "0.034": 7 }
  9. FREQ_DESC: { "Annual": 540 }
 10. TIME_FORMAT_DESC: { "Annual": 540 }
… with 5 more lines</pre>
<p>You can select a subset of features with the <code>-c</code> option, which accepts both integers and column names:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstat</span> <span style="text-decoration: underline">venture.csv</span> -c 3,GEO
  3. "TIME_PERIOD"
 
        Type of data:          Number
        Contains null values:  False
        Unique values:         9
        Smallest value:        2,007
        Largest value:         2,015
        Sum:                   1,085,940
        Mean:                  2,011
        Median:                2,011
        StDev:                 2.584
        Most common values:    2,015 (60x)
                               2,007 (60x)
                               2,008 (60x)
                               2,009 (60x)
                               2,010 (60x)
 
  6. "GEO"
 
        Type of data:          Text
        Contains null values:  False
        Unique values:         20
        Longest value:         2 characters
        Most common values:    CZ (27x)
                               DE (27x)
                               DK (27x)
                               EL (27x)
                               ES (27x)
 
Row count: 540</pre>

<div class="rmdtip">
Keep in mind that <code>csvstat</code>, just like <code>csvsql</code>, employs heuristics to determine the data type, and therefore may not always get it right.
I encourage you to always do a manual inspection as discussed in the previous subsection.
Moreover, even though the type may be a string or integer, that doesn’t say anything about how it should be used.
</div>
<p>As a nice extra, <code>csvstat</code> outputs, at the very end, the number of data points (rows).
Newlines and commas inside values are handled correctly.
To only see that last line, you can use <code>tail</code>.
Alternatively, you can use <code>xsv</code>, which only returns the actual number of rows.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstat</span> <span style="text-decoration: underline">venture.csv</span> | <span style="color: #5f8700">tail</span> -n 1
Row count: 540
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">xsv</span> count <span style="text-decoration: underline">venture.csv</span>
540</pre>
<p>Note that these two options are different from using <code>wc -l</code>, which counts the number of newlines (and therefore also counts the header).</p>
</div>
<div id="r-one-liners-on-the-shell" class="section level3" number="7.3.2">
<h3 number="7.3.2"><span class="header-section-number">7.3.2</span> R One-Liners on the Shell</h3>
<p>In this section I’d like to introduce you to a command-line tool called <code>rush</code><span class="citation"><a href="#fn92" class="footnote-ref" id="fnref92"><sup>92</sup></a></span>,
which enables you to leverage the statistical programming environment <code>R</code><span class="citation"><a href="#fn93" class="footnote-ref" id="fnref93"><sup>93</sup></a></span> directly from the command line.
Before I explain what <code>rush</code> does and why it exists, lets talk a bit about <code>R</code> itself.</p>
<p><code>R</code> is a very powerful statistical software package to do data science.
It’s an interpreted programming language, has an extensive collection of packages, and offers its own REPL, which allows you, similar to the command line, to play with your data.
Note that, once you start R, you’re in an interactive session that is separated from the Unix command line.</p>
<p>Imagine that you have a CSV file called <em>tips.csv</em>, and you would like compute the tip percentage, and save the result.
To accomplish this in <code>R</code> you would first run, <code>R</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">R</span> --quiet <span class="callout">&#10122;</span>
&gt;</pre>
<p><span class="callout">&#10122;</span> I use the <code>--quiet</code> option here to suppress the rather long startup message</p>
<p>And then run the following code:</p>
<pre>&gt; library(tidyverse)                            <span class="callout">&#10122;</span>
── <span style="font-weight: bold">Attaching packages</span> ─────────────────────────────────────── tidyverse 1.3.0 ──
<span style="color: #5f8700">✔</span> <span style="color: #0087ff">ggplot2</span> 3.3.3     <span style="color: #5f8700">✔</span> <span style="color: #0087ff">purrr  </span> 0.3.4
<span style="color: #5f8700">✔</span> <span style="color: #0087ff">tibble </span> 3.0.6     <span style="color: #5f8700">✔</span> <span style="color: #0087ff">dplyr  </span> 1.0.4
<span style="color: #5f8700">✔</span> <span style="color: #0087ff">tidyr  </span> 1.1.2     <span style="color: #5f8700">✔</span> <span style="color: #0087ff">stringr</span> 1.4.0
<span style="color: #5f8700">✔</span> <span style="color: #0087ff">readr  </span> 1.4.0     <span style="color: #5f8700">✔</span> <span style="color: #0087ff">forcats</span> 0.5.1
── <span style="font-weight: bold">Conflicts</span> ────────────────────────────────────────── tidyverse_conflicts() ──
<span style="color: #d70000">✖</span> <span style="color: #0087ff">dplyr</span>::<span style="color: #5f8700">filter()</span> masks <span style="color: #0087ff">stats</span>::filter()
<span style="color: #d70000">✖</span> <span style="color: #0087ff">dplyr</span>::<span style="color: #5f8700">lag()</span>    masks <span style="color: #0087ff">stats</span>::lag()
&gt; df &lt;- read_csv("tips.csv")                    <span class="callout">&#10123;</span>
 
<span style="color: #00afaf">──</span> <span style="font-weight: bold">Column specification</span> <span style="color: #00afaf">────────────────────────────────────────────────────────
</span>cols(
  bill = <span style="color: #5f8700">col_double()</span>,
  tip = <span style="color: #5f8700">col_double()</span>,
  sex = <span style="color: #d70000">col_character()</span>,
  smoker = <span style="color: #d70000">col_character()</span>,
  day = <span style="color: #d70000">col_character()</span>,
  time = <span style="color: #d70000">col_character()</span>,
  size = <span style="color: #5f8700">col_double()</span>
)
 
&gt; df &lt;- mutate(df, percent = tip / bill * 100)  <span class="callout">&#10124;</span>
&gt; write_csv(df, "percent.csv")                  <span class="callout">&#10125;</span>
&gt; q("no")                                       <span class="callout">&#10126;</span>
 
<span style="font-weight: bold">$</span></pre>
<p><span class="callout">&#10122;</span> Load any required packages
<br><span class="callout">&#10123;</span> Read in the CSV file and assign it to a variable
<br><span class="callout">&#10124;</span> Compute a new column <em>percent</em>
<br><span class="callout">&#10125;</span> Save the result to disk
<br><span class="callout">&#10126;</span> Exit <code>R</code></p>
<p>Afterwards, you can continue with the saved file <em>percent.csv</em> on the command line.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">percent.csv</span> <span style="color: #5f8700">trim</span> 5
bill,tip,sex,smoker,day,time,size,percent
16.99,1.01,Female,No,Sun,Dinner,2,5.9446733372572105
10.34,1.66,Male,No,Sun,Dinner,3,16.054158607350097
21.01,3.5,Male,No,Sun,Dinner,3,16.658733936220845
23.68,3.31,Male,No,Sun,Dinner,2,13.97804054054054
… with 240 more lines</pre>
<p>Note that only the third line is associated with what you want to accomplish specifically.
The other lines are necessary boilerplate.
Typing in this boilerplate in order to accomplish something simple is cumbersome and breaks your workflow.
Sometimes, you only want to do one or two things at a time to your data.
Wouldn’t it be great if you could harness the power of <code>R</code> and to use it from the command line?</p>
<p>This is where <code>rush</code> comes in.
Let’s perform the same task as before, but now using <code>rush</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rm</span> <span style="text-decoration: underline">percent.csv</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run -t <span style="color: #af8700">'mutate(df, percent = tip / bill * 100)'</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> percent.csv
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">percent.csv</span> <span style="color: #5f8700">trim</span> 5
bill,tip,sex,smoker,day,time,size,percent
16.99,1.01,Female,No,Sun,Dinner,2,5.9446733372572105
10.34,1.66,Male,No,Sun,Dinner,3,16.054158607350097
21.01,3.5,Male,No,Sun,Dinner,3,16.658733936220845
23.68,3.31,Male,No,Sun,Dinner,2,13.97804054054054
… with 240 more lines</pre>
<p>These small one-liners are possible because <code>rush</code> takes care of all the boilerplate.
In this case I’m using the <code>run</code> subcommand. There’s also the <code>plot</code> subcommand, which I’ll use in the next section to produce data visualizations quickly.
If you’re passing in any input data, then by default, <code>rush</code> assumes that it’s in CSV format with a header and a comma as the delimiter.
Moreover, the column names are sanitized so that they are easier to work with.
You can override these defaults using the <code>--no-header</code> (or <code>-H</code>), <code>--delimiter</code> (or <code>-d</code>), and <code>--no-clean-names</code> (or <code>-C</code>) options, respectively.
The help gives a good overview of the available options for the <code>run</code> subcommand:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run --help
rush: Run an R expression
 
Usage:
  rush run [options] &lt;expression&gt; [--] [&lt;file&gt;...]
 
Reading options:
  -d, --delimiter &lt;str&gt;    Delimiter [default: ,].
  -C, --no-clean-names     No clean names.
  -H, --no-header          No header.
 
Setup options:
  -l, --library &lt;name&gt;     Libraries to load.
  -t, --tidyverse          Enter the Tidyverse.
 
Saving options:
      --dpi &lt;str|int&gt;      Plot resolution [default: 300].
      --height &lt;int&gt;       Plot height.
  -o, --output &lt;str&gt;       Output file.
      --units &lt;str&gt;        Plot size units [default: in].
  -w, --width &lt;int&gt;        Plot width.
 
General options:
  -n, --dry-run            Only print generated script.
  -h, --help               Show this help.
  -q, --quiet              Be quiet.
      --seed &lt;int&gt;         Seed random number generator.
  -v, --verbose            Be verbose.
      --version            Show version.</pre>
<p>Under the hood, <code>rush</code> generates an <code>R</code> script and subsequently executes it.
You can view this generated script by specifying the <code>--dry-run</code> (or <code>-n</code>) option:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run -n --tidyverse <span style="color: #af8700">'mutate(df, percent = tip / bill * 100)'</span> <span style="text-decoration: underline">tips.csv</span>
<span style="font-style: italic"></span><span style="font-style: italic; color: #9e9e9e">#!/usr/bin/env Rscript</span>
<span style="color: #00afaf">library</span><span style="color: #af8700">(</span>tidyverse<span style="color: #af8700">)</span>
<span style="color: #00afaf">library</span><span style="color: #af8700">(</span>glue<span style="color: #af8700">)</span>
df <span style="color: #5f8700">&lt;-</span> janitor::<span style="color: #00afaf">clean_names</span><span style="color: #af8700">(</span>readr::<span style="color: #00afaf">read_delim</span><span style="color: #0087ff">(</span><span style="color: #af8700">"tips.csv"</span>, delim = <span style="color: #af8700">","</span>, col_names
= <span style="color: #0087ff">TRUE)</span><span style="color: #af8700">)</span>
<span style="color: #00afaf">mutate</span><span style="color: #af8700">(</span>df, percent = tip<span style="color: #5f8700">/</span>bill <span style="color: #5f8700">*</span> <span style="color: #0087ff">100</span><span style="color: #af8700">)</span></pre>
<p>This generated script:</p>
<ul>
<li>Writes out the shebang (<em><code>#!</code></em>; see <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>) needed for running an <code>R</code> script from the command line.</li>
<li>Imports the <em>tidyverse</em> and <em>glue</em> packages.</li>
<li>Loads <em>tips.csv</em> as a data frame, cleans the column names, and assigns it to a variable <em>df</em>.</li>
<li>Runs the specified expression.</li>
<li>Prints the result to standard output.</li>
</ul>
<p>You could redirect this generated script to a file and easily turn it into a new command-line tool because of the shebang.</p>
<p>The output of <code>rush</code> doesn’t have to be a in CSV format per se. Here, I compute the mean tip percent, the maximum party size, the unique values of the time column, the correlation between the bill and the tip. Finally, I extract an entire column (but only show the first 10 values).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">percent.csv</span> <span style="color: #5f8700">rush</span> run <span style="color: #af8700">'mean(df$percent)'</span> -
16.0802581722505
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">percent.csv</span> <span style="color: #5f8700">rush</span> run <span style="color: #af8700">'max(df$size)'</span> -
6
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">percent.csv</span> <span style="color: #5f8700">rush</span> run <span style="color: #af8700">'unique(df$time)'</span> -
Dinner
Lunch
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">percent.csv</span> <span style="color: #5f8700">rush</span> run <span style="color: #af8700">'cor(df$bill, df$tip)'</span> -
0.675734109211365
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">percent.csv</span> <span style="color: #5f8700">rush</span> run <span style="color: #af8700">'df$tip'</span> - | <span style="color: #5f8700">trim</span>
1.01
1.66
3.5
3.31
3.61
4.71
2
3.12
1.96
3.23
… with 234 more lines</pre>
<p>That last dash means that <code>rush</code> should read from standard input.</p>
<p>So now, if you want to do one or two things to your data set with <code>R</code>, you can specify it as a one-liner, and keep on working on the command line.
All the knowledge that you already have about <code>R</code> can now be used from the command line. With <code>rush</code>, you can even create sophisticated visualizations, as I’ll show you in the next section.</p>
</div>
</div>
<div id="creating-visualizations" class="section level2" number="7.4">
<h2 number="7.4"><span class="header-section-number">7.4</span> Creating Visualizations</h2>
<p>In this section, I’m going to show you how to create data visualizations at the command line.
Using <code>rush plot</code> I’ll be creating bar charts, scatter plots, and box plots.
Before we dive in, though, I’d first like to explain how you can display your visualizations.</p>
<div id="displaying-images-from-the-command-line" class="section level3" number="7.4.1">
<h3 number="7.4.1"><span class="header-section-number">7.4.1</span> Displaying Images from the Command Line</h3>
<p>Let’s take the image <em>tips.png</em> as an example.
Take a look at Figure @ref(fig:plot-demo), which is a data visualization created using <code>rush</code> and the <em>tips.csv</em> dataset.
(I’ll explain the <code>rush</code> syntax in a moment.)
I use the <code>display</code> tool to insert the image in the book, but if you run <code>display</code> you’ll find that it doesn’t
work.
That’s because displaying images from the command line is actually quite tricky.</p>
<div class="figure" style="text-align: center">
<img src="images/tips.png" alt="Displaying this image yourself can be tricky" width="90%" />
<p class="caption">
(#fig:plot-demo)Displaying this image yourself can be tricky
</p>
</div>
<p>Depending on your setup, there are different options available to display images.
I know of four options, each with their own advantages and disadvantages:
(1) as a textual representation,
(2) as an inline image,
(3) using an image viewer, and
(4) using a browser.
Let’s go through them quickly.</p>
<div class="figure" style="text-align: center">
<img src="images/screenshot_display_ansi_and_inline.png" alt="Displaying an image in the terminal via ASCII characters and ANSI escape sequences (top) and via the iTerm2 inline images protocol (bottom)" width="90%" />
<p class="caption">
(#fig:screenshot-display-ansi-and-inline)Displaying an image in the terminal via ASCII characters and ANSI escape sequences (top) and via the iTerm2 inline images protocol (bottom)
</p>
</div>
<p>Option 1 is to display the image inside the terminal as shown at the top of Figure @ref(fig:screenshot-display-ansi-and-inline).
This output is generated by <code>rush</code> when the standard output is not redirected to a file.
It’s based on ASCII characters and ANSI escape sequences, so it’s available in every terminal.
Depending on how you’re reading this book, the output you get when you run this code might or might not match the screenshot in Figure @ref(fig:screenshot-display-ansi-and-inline).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x bill --y tip --color size --facets <span style="color: #af8700">'~day'</span> <span style="text-decoration: underline">tips.csv</span>              
<span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #1c1c1c; background: #ffffff">Fri</span><span style="color: #ffffff; background: #ffffff">                               </span><span style="color: #1c1c1c; background: #ffffff">Sat</span><span style="color: #ffffff; background: #ffffff">                     
  </span><span style="color: #4e4e4e; background: #ffffff">10.0</span><span style="color: #ffffff; background: #ffffff">                                                                  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">     
   </span><span style="color: #4e4e4e; background: #ffffff">7.5</span><span style="color: #ffffff; background: #ffffff">                                                            </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">             
                                                           </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">            </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">       
   </span><span style="color: #4e4e4e; background: #ffffff">5.0</span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">               </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                   </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">###</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">               
            </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">###</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">                       </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">###</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">##</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #ffffff; background: #5fafd7"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">          
   </span><span style="color: #4e4e4e; background: #ffffff">2.5</span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #005f5f; background: #ffffff"></span><span style="color: #005f5f; background: #005f5f">%</span><span style="color: #5f5f87; background: #005f5f"></span><span style="color: #5f5f87; background: #5f5f87">###</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">                           </span><span style="color: #005f5f; background: #ffffff"></span><span style="color: #005f5f; background: #005f5f">%</span><span style="color: #ffffff; background: #005f5f"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#########</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">                 
</span><span style="color: #080808; background: #ffffff">t</span><span style="color: #ffffff; background: #ffffff">                                                                          </span><span style="color: #080808; background: #ffffff">size</span><span style="color: #ffffff; background: #ffffff"> 
</span><span style="color: #080808; background: #ffffff">i</span><span style="color: #ffffff; background: #ffffff">                     </span><span style="color: #1c1c1c; background: #ffffff">Sun</span><span style="color: #ffffff; background: #ffffff">                               </span><span style="color: #1c1c1c; background: #ffffff">Thur</span><span style="color: #ffffff; background: #ffffff">                </span><span style="color: #080808; background: #ffffff">6</span><span style="color: #ffffff; background: #ffffff">   
</span><span style="color: #080808; background: #ffffff">p</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">10.0</span><span style="color: #ffffff; background: #ffffff">                                                                      </span><span style="color: #080808; background: #ffffff">1</span><span style="color: #ffffff; background: #ffffff">   
   </span><span style="color: #4e4e4e; background: #ffffff">7.5</span><span style="color: #ffffff; background: #ffffff">                                                         </span><span style="color: #87afff; background: #ffffff"></span><span style="color: #87afff; background: #87afff">=</span><span style="color: #ffffff; background: #87afff"></span><span style="color: #ffffff; background: #ffffff">                
                    </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                           </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                
   </span><span style="color: #4e4e4e; background: #ffffff">5.0</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5fafd7; background: #ffffff"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #5f87af; background: #5fafd7"></span><span style="color: #5f87af; background: #5f87af">#*#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5fafd7; background: #ffffff"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #5f87af; background: #5fafd7"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #87afff; background: #ffffff"></span><span style="color: #87afff; background: #87afff">=</span><span style="color: #ffffff; background: #87afff"></span><span style="color: #ffffff; background: #ffffff">             </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #87afff; background: #ffffff"></span><span style="color: #87afff; background: #87afff">=</span><span style="color: #ffffff; background: #87afff"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #87afff; background: #ffffff"></span><span style="color: #87afff; background: #87afff">=</span><span style="color: #ffffff; background: #87afff"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5fafd7; background: #ffffff"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #5f87af; background: #5fafd7"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">           
   </span><span style="color: #4e4e4e; background: #ffffff">2.5</span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">##</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">###</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">            </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">###</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">                    
           </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #ffffff; background: #5fafd7"></span><span style="color: #ffffff; background: #ffffff">                   </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">######</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                           
           </span><span style="color: #4e4e4e; background: #ffffff">10</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #4e4e4e; background: #ffffff">20</span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #4e4e4e; background: #ffffff">30</span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #4e4e4e; background: #ffffff">40</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #4e4e4e; background: #ffffff">50</span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #4e4e4e; background: #ffffff">10</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #4e4e4e; background: #ffffff">20</span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #4e4e4e; background: #ffffff">30</span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #4e4e4e; background: #ffffff">40</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #4e4e4e; background: #ffffff">50</span><span style="color: #ffffff; background: #ffffff">      
                                      </span><span style="color: #080808; background: #ffffff">bill</span><span style="color: #ffffff; background: #ffffff">                                      </span></pre>
<p>If you only see ASCII characters, that means the medium on which you’re reading this book doesn’t support the ANSI escape sequences responsible for the colors.
Fortunately, if you run the above command yourself, it will look just like the screenshot.</p>
<p>Option 2, as seen at the bottom of Figure @ref(fig:screenshot-display-ansi-and-inline), also displays images inside the terminal.
This is the iTerm2 terminal, which is only available for macOS and uses the <a href="https://iterm2.com/documentation-images.html">Inline Images Protocol</a> through a small script (which I have named <code>display</code>).
This script is not included with the Docker image, but you can easily install it:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://iterm2.com/utilities/imgcat"</span> <span style="color: #af8700">&gt;</span> display &amp;&amp; <span style="color: #5f8700">chmod</span> u+x display</pre>
<p>If you’re not using iTerm2 on macOS, there might be other options available to display images inline.
Please consult your favorite search engine.</p>
<div class="figure" style="text-align: center">
<img src="images/screenshot_display_preview_and_browser.png" alt="Displaying an image externally via a file explorer and an image viewer (left) and via a webserver and a browser (right)" width="90%" />
<p class="caption">
(#fig:screenshot-display-preview-and-browser)Displaying an image externally via a file explorer and an image viewer (left) and via a webserver and a browser (right)
</p>
</div>
<p>Option 3 is to manually open the image (<em>tips.csv</em> in this example) in an image viewer.
Figure @ref(fig:screenshot-display-preview-and-browser) shows, on the left, the file explorer (Finder) and image viewer (Preview) on macOS.
When you’re working locally, this option always works.
When you’re working inside a Docker container, you can only access the generated image from your OS when you’ve mapped a local directory using the <code>-v</code> option.
See <a href="#chapter-2-getting-started">Chapter 2</a> for instructions on how to do this.
An advantage of this option is that most image viewers automatically update the display when the image has changed, which allows for quick iterations as you fine-tune your visualization.</p>
<p>Option 4 is to open the image in a browser.
The right side of Figure @ref(fig:screenshot-display-preview-and-browser) is a screenshot of Firefox showing <em><a href="http://localhost:8000/tips.png" class="uri">http://localhost:8000/tips.png</a></em>.
Any browser will do, but you need two other prerequisites for this option to work.
First, you need to have made a port (port 8000 in this example) accessible on the Docker container using the <code>-p</code> option.
(Again, see <a href="#chapter-2-getting-started">Chapter 2</a> for instructions on how to do this.)
Second, you need to start a webserver.
For this, the Docker container has a small tool called <code>servewd</code><span class="citation"><a href="#fn94" class="footnote-ref" id="fnref94"><sup>94</sup></a></span>, which serves the current working directory using Python:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="color: #af005f">$(</span><span style="color: #5f8700">which</span> servewd<span style="color: #af005f">)</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">/usr/bin/dsutils/servewd</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env bash</span>
   2   │ ARGS<span style="color: #af005f">=</span><span style="color: #5f8700">"$@"</span>
   3   │ python3 -m http.server ${ARGS} <span style="color: #af8700">2</span><span style="color: #af005f">&gt;</span>/dev/null <span style="color: #af005f">&amp;</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>You only need to run <code>servewd</code> once from a directory (for example, <em>/data/</em>) and it will happily run in the background.
Once you’ve plotted something, you can visit <em>localhost:8000</em> in your browser and access the contents of that directory and all of its subdirectories.
The default port is 8000, but you can change this by specifying it as an argument to <code>servewd</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">servewd</span> 9999</pre>
<p>Just make sure that this port is accessible.
Because <code>servewd</code> runs in the background, you need to stop it as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">pkill</span> -f http.server</pre>
<p>Option 4 can also work on a remote machine.</p>
<p>Now that we’ve covered four options to display images, let’s move on to actually creating some.</p>
</div>
<div id="plotting-in-a-rush" class="section level3" number="7.4.2">
<h3 number="7.4.2"><span class="header-section-number">7.4.2</span> Plotting in a Rush</h3>
<p>When it comes to creating data visualizations, there’s a plethora of options.
Personally, I’m a staunch proponent of <code>ggplot2</code>, which is a visualization package for R.
The underlying grammar of graphics is accompanied by a consistent API that allows you to quickly and iteratively create different types of beautiful data visualizations while rarely having to consult the documentation.
A welcoming set of properties when exploring data.</p>
<p>We’re not really in a rush, but we also don’t want to fiddle too much about any single visualization.
Moreover, we’d like to stay at the command line as much as possible.
Luckily, we still have <code>rush</code>, which allows us to <code>ggplot2</code> from the command line.
The data visualization from Figure @ref(fig:plot-demo) could have been created as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run --library ggplot2 <span style="color: #af8700">'ggplot(df, aes(x = bill, y = tip, color = size)) +
 geom_point() + facet_wrap(~day)'</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> tips.png</pre>
<p>However, as you may have noticed, I have used a very different command to create <em>tips.png</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x bill --y tip --color size --facets <span style="color: #af8700">'~day'</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> tips.png</pre>
<p>While the syntax of <code>ggplot2</code> is relatively concise, especially considering the flexibility it offers, there’s a shortcut to create basic plots quickly.
This shortcut is available through the <code>plot</code> subcommand of <code>rush</code>.
This allows you to create beautiful basic plots without needing to learn R and the grammar of graphics.</p>
<p>Under the hood, <code>rush plot</code> uses the function <code>qplot</code> from the <code>ggplot2</code> package.
Here’s the first part of its documentation:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">R</span> -q -e <span style="color: #af8700">'?ggplot2::qplot'</span> | <span style="color: #5f8700">trim</span> 14
&gt; ?ggplot2::qplot
qplot                 package:ggplot2                  R Documentation
 
Quick plot
 
Description:
 
     ‘qplot()’ is a shortcut designed to be familiar if you're used to
     base ‘plot()’. It's a convenient wrapper for creating a number of
     different types of plots using a consistent calling scheme. It's
     great for allowing you to produce plots quickly, but I highly
     recommend learning ‘ggplot()’ as it makes it easier to create
     complex graphics.
 
… with 108 more lines</pre>
<p>I agree with this advice; once you’re done reading this book, it’ll be worthwhile to learn <code>ggplot2</code>, especially if you want to upgrade any exploratory data visualizations into ones that are suitable for communication.
For now, while we’re at the command line, let’s take that shortcut.</p>
<p>As Figure @ref(fig:screenshot-display-ansi-and-inline) already showed, <code>rush plot</code> can create both graphical visualizations (consisting of pixels) and textual visualizations (consisting of ASCII characters and ANSI escape sequences) with the same syntax.
When <code>rush</code> detects that its output is piped to another command (such as <code>display</code> or redirected to a file such as <em>tips.png</em> it will produce a graphical visualization; otherwise it will produce a textual visualization.</p>
<p>Let’s take a moment to read through the plotting and saving options of <code>rush plot</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --help
rush: Quick plot
 
Usage:
  rush plot [options] [--] [&lt;file&gt;|-]
 
Reading options:
  -d, --delimiter &lt;str&gt;    Delimiter [default: ,].
  -C, --no-clean-names     No clean names.
  -H, --no-header          No header.
 
Setup options:
  -l, --library &lt;name&gt;     Libraries to load.
  -t, --tidyverse          Enter the Tidyverse.
 
Plotting options:
      --aes &lt;key=value&gt;    Additional aesthetics.
  -a, --alpha &lt;name&gt;       Alpha column.
  -c, --color &lt;name&gt;       Color column.
      --facets &lt;formula&gt;   Facet specification.
  -f, --fill &lt;name&gt;        Fill column.
  -g, --geom &lt;geom&gt;        Geometry [default: auto].
      --group &lt;name&gt;       Group column.
      --log &lt;x|y|xy&gt;       Variables to log transform.
      --margins            Display marginal facets.
      --post &lt;code&gt;        Code to run after plotting.
      --pre &lt;code&gt;         Code to run before plotting.
      --shape &lt;name&gt;       Shape column.
      --size &lt;name&gt;        Size column.
      --title &lt;str&gt;        Plot title.
  -x, --x &lt;name&gt;           X column.
      --xlab &lt;str&gt;         X axis label.
  -y, --y &lt;name&gt;           Y column.
      --ylab &lt;str&gt;         Y axis label.
  -z, --z &lt;name&gt;           Z column.
 
Saving options:
      --dpi &lt;str|int&gt;      Plot resolution [default: 300].
      --height &lt;int&gt;       Plot height.
  -o, --output &lt;str&gt;       Output file.
      --units &lt;str&gt;        Plot size units [default: in].
  -w, --width &lt;int&gt;        Plot width.
 
General options:
  -n, --dry-run            Only print generated script.
  -h, --help               Show this help.
  -q, --quiet              Be quiet.
      --seed &lt;int&gt;         Seed random number generator.
  -v, --verbose            Be verbose.
      --version            Show version.</pre>
<p>The most important options are the plotting options that take a <em><code>&lt;name&gt;</code></em> as an argument.
For example, the <code>--x</code> option allows you to specify which column should be used to determine where <em>things</em> should be placed along the x axis.
The same holds for the <code>--y</code> option.
The <code>--color</code> and <code>--fill</code> options are used to specify which column you want to use for coloring.
You can probably guess what the <code>--size</code> and <code>--alpha</code> options are about.
Other common options are explained throughout the sections as I create various visualizations.
Note that for each visualization, I first show its textual representation (ASCII and ANSI characters) and then its visual representation (pixels).</p>
</div>
<div id="creating-bar-charts" class="section level3" number="7.4.3">
<h3 number="7.4.3"><span class="header-section-number">7.4.3</span> Creating Bar Charts</h3>
<p>Bar charts are especially useful for displaying the value counts of a categorical feature.
Here’s a textual visualization of the <em>time</em> feature in the tips dataset:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x time <span style="text-decoration: underline">tips.csv</span>           
<span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">                                       
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">                                       
  </span><span style="color: #4e4e4e; background: #ffffff">150</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">                                       
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">                                       
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">                                       
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">                                       
  </span><span style="color: #4e4e4e; background: #ffffff">100</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">                                       
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">                                       
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">     
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">     
   </span><span style="color: #4e4e4e; background: #ffffff">50</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">     
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">     
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">     
         </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">     
    </span><span style="color: #4e4e4e; background: #ffffff">0</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #585858; background: #ffffff"></span><span style="color: #585858; background: #585858">********************************</span><span style="color: #ffffff; background: #585858"></span><span style="color: #ffffff; background: #ffffff">     
                      </span><span style="color: #4e4e4e; background: #ffffff">Dinner</span><span style="color: #ffffff; background: #ffffff">                            </span><span style="color: #4e4e4e; background: #ffffff">Lunch</span><span style="color: #ffffff; background: #ffffff">                   
                                        </span><span style="color: #080808; background: #ffffff">time</span><span style="color: #ffffff; background: #ffffff">                                    </span></pre>
<p>Figure @ref(fig:plot-bar-image) shows the graphical visualization, which is created by <code>rush plot</code> when the output is redirected to a file.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x time <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> plot-bar.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">plot-bar.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/plot-bar.png" alt="A bar chart" width="90%" />
<p class="caption">
(#fig:plot-bar-image)A bar chart
</p>
</div>
<p>The conclusion we can draw from this bar chart is straightforward: there are more than twice as many data points for dinner than lunch.</p>
</div>
<div id="creating-histograms" class="section level3" number="7.4.4">
<h3 number="7.4.4"><span class="header-section-number">7.4.4</span> Creating Histograms</h3>
<p>The counts of a continuous variable can be visualized with a histogram.
Here, I have used the <em>time</em> feature to set the fill color.
As a result, <code>rush plot</code> conveniently creates a stacked histogram.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x tip --fill time <span style="text-decoration: underline">tips.csv
</span><span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">             </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                                                
             </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                                        
  </span><span style="color: #4e4e4e; background: #ffffff">40</span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                                        
             </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                                        
             </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                                        
  </span><span style="color: #4e4e4e; background: #ffffff">30</span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                                        
             </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                                 </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #ff8787">t</span><span style="color: #080808; background: #ffffff">ime</span><span style="color: #ffffff; background: #ffffff">   
           </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=====</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                                 </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #00d7d7">D</span><span style="color: #080808; background: #ffffff">inner</span><span style="color: #ffffff; background: #ffffff"> 
  </span><span style="color: #4e4e4e; background: #ffffff">20</span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=====</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                               </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #00d7d7">L</span><span style="color: #080808; background: #ffffff">unch</span><span style="color: #ffffff; background: #ffffff">  
           </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">====</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=====</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                          </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">      
           </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">====</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=====</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                           
  </span><span style="color: #4e4e4e; background: #ffffff">10</span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+++++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">==========</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                           
       </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">====</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++++++++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">====</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=====</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                         
       </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++++++++++++++++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">======</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">======</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                              
   </span><span style="color: #4e4e4e; background: #ffffff">0</span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">         
                 </span><span style="color: #4e4e4e; background: #ffffff">2.5</span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #4e4e4e; background: #ffffff">5.0</span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #4e4e4e; background: #ffffff">7.5</span><span style="color: #ffffff; background: #ffffff">             </span><span style="color: #4e4e4e; background: #ffffff">10.0</span><span style="color: #ffffff; background: #ffffff">         
                                     </span><span style="color: #080808; background: #ffffff">tip</span><span style="color: #ffffff; background: #ffffff">                                        </span></pre>
<p>Figure @ref(fig:plot-histogram-image) shows the graphical visualization.</p>

<div class="rmdtip">
Allow me to demonstrate two syntax shortcuts that you may find useful.
The two exclamation marks (<code>!!</code>) get replaced with the previous command.
The exclamation mark and dollar sign (<code>!$</code>) get replaced by the last part of the previous command, which is the filename <em>plot-histogram.png</em>.
As you can see, the updated commands are first printed by the Z shell so you know exactly what it executes.
These two shortcuts can save a lot of typing, but they’re not easy to remember.
</div>
<pre><span style="font-weight: bold">$</span> <span style="color: #0087ff">!!</span> <span style="color: #af8700">&gt;</span> plot-histogram.png
rush plot --x tip --fill time tips.csv &gt; plot-histogram.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="color: #0087ff">!$</span>
display plot-histogram.png</pre>
<div class="figure" style="text-align: center">
<img src="images/plot-histogram.png" alt="A histogram" width="90%" />
<p class="caption">
(#fig:plot-histogram-image)A histogram
</p>
</div>
<p>This histogram reveals that most tips are around 2,5 USD.
Because the two groups dinner and lunch groups are stacked on top of each other and show absolute counts, it’s difficult to compare them.
Perhaps a density plot can help with this.</p>
</div>
<div id="creating-density-plots" class="section level3" number="7.4.5">
<h3 number="7.4.5"><span class="header-section-number">7.4.5</span> Creating Density Plots</h3>
<p>A density plot is useful for visualizing the distribution of a continuous variable.
<code>rush plot</code> uses heuristics to determine the appropriate geometry, but you can override this with the <code>geom</code> option:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x tip --fill time --geom density <span style="text-decoration: underline">tips.csv</span>                         
<span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #4e4e4e; background: #ffffff">0.5</span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                                                
            </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                                               
  </span><span style="color: #4e4e4e; background: #ffffff">0.4</span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">+++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                                              
           </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                                              
           </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">+++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                                         
  </span><span style="color: #4e4e4e; background: #ffffff">0.3</span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@@@</span><span style="color: #ff8787; background: #080808"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #080808; background: #ff8787"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                                       
          </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ff8787; background: #080808"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #080808; background: #ff8787"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                               </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">t</span><span style="color: #080808; background: #ffffff">ime</span><span style="color: #ffffff; background: #ffffff">   
          </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">+++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ff8787; background: #080808"></span><span style="color: #ff8787; background: #ff8787">====</span><span style="color: #080808; background: #ff8787"></span><span style="color: #080808; background: #080808">@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                              </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">D</span><span style="color: #080808; background: #ffffff">inner</span><span style="color: #ffffff; background: #ffffff"> 
  </span><span style="color: #4e4e4e; background: #ffffff">0.2</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">+++++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@@</span><span style="color: #ff8787; background: #080808"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #080808; background: #ff8787"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                            </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">L</span><span style="color: #080808; background: #ffffff">unch</span><span style="color: #ffffff; background: #ffffff">  
         </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">+++++++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@@</span><span style="color: #ff8787; background: #080808"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #080808; background: #ff8787"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                           </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">      
        </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">++++++++++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@@@@@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                                
  </span><span style="color: #4e4e4e; background: #ffffff">0.1</span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">@@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">+++++++++++++++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@@@@@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                           
       </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">++++++++++++++++++++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@</span><span style="color: #00d7d7; background: #080808"></span><span style="color: #00d7d7; background: #00d7d7">++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                                        
       </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">++++++++++++++++++++++++++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@@@@@@@@@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">                              
  </span><span style="color: #4e4e4e; background: #ffffff">0.0</span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">++++++++++++++++++++++++++++++++++++++++++</span><span style="color: #080808; background: #00d7d7"></span><span style="color: #080808; background: #080808">@@@@@@@@@@@@@@@@@@@@@@</span><span style="color: #ffffff; background: #080808"></span><span style="color: #ffffff; background: #ffffff">         
                 </span><span style="color: #4e4e4e; background: #ffffff">2.5</span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #4e4e4e; background: #ffffff">5.0</span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #4e4e4e; background: #ffffff">7.5</span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #4e4e4e; background: #ffffff">10.0</span><span style="color: #ffffff; background: #ffffff">        
                                     </span><span style="color: #080808; background: #ffffff">tip</span><span style="color: #ffffff; background: #ffffff">                                        </span></pre>
<p>In this case, the textual representation really shows its limitations when compared to the visual representation in Figure @ref(fig:plot-density-image).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x tip --fill time --geom density <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> plot-density.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">plot-density.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/plot-density.png" alt="A density plot" width="90%" />
<p class="caption">
(#fig:plot-density-image)A density plot
</p>
</div>
</div>
<div id="happy-little-accidents" class="section level3" number="7.4.6">
<h3 number="7.4.6"><span class="header-section-number">7.4.6</span> Happy Little Accidents</h3>
<p>You’ve already seen three types of visualizations.
In <code>ggplot2</code>, these correspond to the functions <code>geom_bar</code>, <code>geom_histogram</code>, and <code>geom_density</code>.
<em>geom</em> is short for geometry and dictates what is actually being plotted.
This <a href="https://ggplot2.tidyverse.org/">cheat sheet for <code>ggplot2</code></a> provides a good overview of the available geometry types.
Which geometry types you can use depend on the columns (and their types) you specify.
Not every combination makes sense.
Take this line plot for example.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x tip --y bill --color size --size day --geom path <span style="text-decoration: underline">tips.csv</span>       
<span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #4e4e4e; background: #ffffff">50</span><span style="color: #ffffff; background: #ffffff">                                           </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">               </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">####</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">       
                         </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #87afff; background: #ffffff"></span><span style="color: #87afff; background: #87afff">==</span><span style="color: #ffffff; background: #87afff"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">***</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">            </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">****#####</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">           
                 </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">###</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #87afff; background: #ffffff"></span><span style="color: #87afff; background: #87afff">=</span><span style="color: #5f87af; background: #87afff"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #ffffff; background: #5fafd7"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">***</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*****####</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                
  </span><span style="color: #4e4e4e; background: #ffffff">40</span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">###***</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #87afff; background: #5fafd7"></span><span style="color: #87afff; background: #87afff">======</span><span style="color: #ffffff; background: #87afff"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*****###</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                     
                   </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#*#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">###</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">******###</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #5f5f87; background: #5fafd7"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #87afff; background: #ffffff"></span><span style="color: #87afff; background: #87afff">======</span><span style="color: #5f87af; background: #87afff"></span><span style="color: #5f87af; background: #5f87af">****###</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                    </span><span style="color: #080808; background: #ffffff">day</span><span style="color: #ffffff; background: #ffffff">  
        </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">################</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*****</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #5f87af; background: #5fafd7"></span><span style="color: #5f87af; background: #5f87af">*##*****</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #87afff; background: #5f5f87"></span><span style="color: #87afff; background: #87afff">=</span><span style="color: #5f87af; background: #87afff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                         </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">F</span><span style="color: #080808; background: #ffffff">ri</span><span style="color: #ffffff; background: #ffffff">  
</span><span style="color: #080808; background: #ffffff">b</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">30</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">+++</span><span style="color: #5f87af; background: #5fafd7"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #87afff; background: #5f5f87"></span><span style="color: #87afff; background: #87afff">==</span><span style="color: #5f87af; background: #87afff"></span><span style="color: #5f87af; background: #5f87af">****</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">+</span><span style="color: #5f87af; background: #5fafd7"></span><span style="color: #5f87af; background: #5f87af">****</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                            </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">S</span><span style="color: #080808; background: #ffffff">at</span><span style="color: #ffffff; background: #ffffff">  
</span><span style="color: #080808; background: #ffffff">i</span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*##</span><span style="color: #5fafd7; background: #5f87af"></span><span style="color: #5fafd7; background: #5fafd7">++</span><span style="color: #5f87af; background: #5fafd7"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">###</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #87afff; background: #5f5f87"></span><span style="color: #87afff; background: #87afff">===</span><span style="color: #5f87af; background: #87afff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #005f5f; background: #5f87af"></span><span style="color: #005f5f; background: #005f5f">%</span><span style="color: #87afff; background: #005f5f"></span><span style="color: #87afff; background: #87afff">=====</span><span style="color: #5f5f87; background: #87afff"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">**#</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                             </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">S</span><span style="color: #080808; background: #ffffff">un</span><span style="color: #ffffff; background: #ffffff">  
</span><span style="color: #080808; background: #ffffff">l</span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">##</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">***</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #87afff; background: #5f5f87"></span><span style="color: #87afff; background: #87afff">=</span><span style="color: #5f5f87; background: #87afff"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #87afff; background: #5f5f87"></span><span style="color: #87afff; background: #87afff">==</span><span style="color: #5f5f87; background: #87afff"></span><span style="color: #5f5f87; background: #5f5f87">##########</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                            </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #080808">T</span><span style="color: #080808; background: #ffffff">hur</span><span style="color: #ffffff; background: #ffffff"> 
</span><span style="color: #080808; background: #ffffff">l</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">20</span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">########</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">**</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">****</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#####</span><span style="color: #5fafd7; background: #5f5f87"></span><span style="color: #5fafd7; background: #5fafd7">++</span><span style="color: #ffffff; background: #5fafd7"></span><span style="color: #ffffff; background: #ffffff">                                            
       </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">##</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#####</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">###*#</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">#####</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                                       </span><span style="color: #080808; background: #ffffff">size</span><span style="color: #ffffff; background: #ffffff"> 
       </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#########</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">##</span><span style="color: #5f5f87; background: #5f87af"></span><span style="color: #5f5f87; background: #5f5f87">###########</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #5f87af; background: #ffffff"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                                      </span><span style="color: #080808; background: #ffffff">5</span><span style="color: #ffffff; background: #ffffff">    
  </span><span style="color: #4e4e4e; background: #ffffff">10</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">###########</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">####</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #5f87af; background: #5f5f87"></span><span style="color: #5f87af; background: #5f87af">*</span><span style="color: #ffffff; background: #5f87af"></span><span style="color: #ffffff; background: #ffffff">                                          
       </span><span style="color: #005f5f; background: #ffffff"></span><span style="color: #005f5f; background: #005f5f">%%</span><span style="color: #5f5f87; background: #005f5f"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">##</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #5f5f87; background: #ffffff"></span><span style="color: #5f5f87; background: #5f5f87">#</span><span style="color: #ffffff; background: #5f5f87"></span><span style="color: #ffffff; background: #ffffff">                                          
       </span><span style="color: #005f5f; background: #ffffff"></span><span style="color: #005f5f; background: #005f5f">%</span><span style="color: #ffffff; background: #005f5f"></span><span style="color: #ffffff; background: #ffffff">                                                                        
                </span><span style="color: #4e4e4e; background: #ffffff">2.5</span><span style="color: #ffffff; background: #ffffff">               </span><span style="color: #4e4e4e; background: #ffffff">5.0</span><span style="color: #ffffff; background: #ffffff">               </span><span style="color: #4e4e4e; background: #ffffff">7.5</span><span style="color: #ffffff; background: #ffffff">               </span><span style="color: #4e4e4e; background: #ffffff">10.0</span><span style="color: #ffffff; background: #ffffff">      
                                      </span><span style="color: #080808; background: #ffffff">tip</span><span style="color: #ffffff; background: #ffffff">                                       </span></pre>
<p>This happy little accident becomes clearer in the visual representation in Figure @ref(fig:plot-accident-image).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x tip --y bill --color size --size day --geom path <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> plot
-accident.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">plot-accident.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/plot-accident.png" alt="A happy little accident" width="90%" />
<p class="caption">
(#fig:plot-accident-image)A happy little accident
</p>
</div>
<p>The rows in <em>tips.csv</em> are independent observations, whereas drawing a line between the data points assumes that they are connected.
It’s better to visualize the relationship between the <em>tip</em> and the <em>bill</em> with a scatter plot.</p>
</div>
<div id="creating-scatter-plots" class="section level3" number="7.4.7">
<h3 number="7.4.7"><span class="header-section-number">7.4.7</span> Creating Scatter Plots</h3>
<p>A scatter plot, where the geometry is a point, happens to be the default when specifying two continuous features:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x bill --y tip --color time <span style="text-decoration: underline">tips.csv</span>                              
<span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #4e4e4e; background: #ffffff">10.0</span><span style="color: #ffffff; background: #ffffff">                                                                </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">         
                                                                   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">            
                                                                                
                                                                                
   </span><span style="color: #4e4e4e; background: #ffffff">7.5</span><span style="color: #ffffff; background: #ffffff">                                                 </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                        
                                  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">                </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">             
</span><span style="color: #080808; background: #ffffff">t</span><span style="color: #ffffff; background: #ffffff">                                   </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                          </span><span style="color: #080808; background: #ffffff">time</span><span style="color: #ffffff; background: #ffffff">   
</span><span style="color: #080808; background: #ffffff">i</span><span style="color: #ffffff; background: #ffffff">             </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">                       </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #ff8787">D</span><span style="color: #080808; background: #ffffff">inner</span><span style="color: #ffffff; background: #ffffff"> 
</span><span style="color: #080808; background: #ffffff">p</span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #4e4e4e; background: #ffffff">5.0</span><span style="color: #ffffff; background: #ffffff">                         </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #00d7d7">L</span><span style="color: #080808; background: #ffffff">unch</span><span style="color: #ffffff; background: #ffffff">  
                 </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                          
                   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">========</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">             </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                
                  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=====</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=====</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">++</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                  
   </span><span style="color: #4e4e4e; background: #ffffff">2.5</span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">++++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">====</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                   
              </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #00d7d7; background: #ffffff"></span><span style="color: #00d7d7; background: #00d7d7">+</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">==</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">++</span><span style="color: #ffffff; background: #00d7d7"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                         
        </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                    </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">                                
                 </span><span style="color: #4e4e4e; background: #ffffff">10</span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #4e4e4e; background: #ffffff">20</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">30</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">40</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">50</span><span style="color: #ffffff; background: #ffffff">          
                                     </span><span style="color: #080808; background: #ffffff">bill</span><span style="color: #ffffff; background: #ffffff">                                       </span></pre>
<p>Note that the color of each point is specified with the <code>--color</code> option (and not with the <code>--fill</code> option).
See Figure @ref(fig:plot-scatter-image) for the visual representation.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x bill --y tip --color time <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> plot-scatter.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">plot-scatter.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/plot-scatter.png" alt="A scatter plot" width="90%" />
<p class="caption">
(#fig:plot-scatter-image)A scatter plot
</p>
</div>
<p>From this scatter plot we may conclude that there’s a relationship between the amount of the bill and the tip.
Perhaps we it’s useful to examine this data from a higher level by creating trend lines.</p>
</div>
<div id="creating-trend-lines" class="section level3" number="7.4.8">
<h3 number="7.4.8"><span class="header-section-number">7.4.8</span> Creating Trend Lines</h3>
<p>If you override the default geometry with <em><code>smooth</code></em>, you can visualize trend lines.
These are useful for seeing the bigger picture.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x bill --y tip --color time --geom smooth <span style="text-decoration: underline">tips.csv</span>                
<span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">                                                                     </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">==</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">         
                                                                   </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">====</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">         
  </span><span style="color: #4e4e4e; background: #ffffff">7.5</span><span style="color: #ffffff; background: #ffffff">                                                            </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">===</span><span style="color: #ff8787; background: #949494"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #ffffff; background: #ff8787"></span><span style="color: #ffffff; background: #ffffff">         
                                                               </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">==</span><span style="color: #ff8787; background: #949494"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #949494; background: #ff8787"></span><span style="color: #949494; background: #949494">===</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">         
                                                     </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">=========</span><span style="color: #ff8787; background: #949494"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #949494; background: #ff8787"></span><span style="color: #949494; background: #949494">======</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">         
                                             </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">=======</span><span style="color: #00d7d7; background: #949494"></span><span style="color: #00d7d7; background: #00d7d7">+++++++++</span><span style="color: #ff8787; background: #00d7d7"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #949494; background: #ff8787"></span><span style="color: #949494; background: #949494">=====</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">             
</span><span style="color: #080808; background: #ffffff">t</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">5.0</span><span style="color: #ffffff; background: #ffffff">                                  </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">====</span><span style="color: #00d7d7; background: #949494"></span><span style="color: #00d7d7; background: #00d7d7">+++++++++</span><span style="color: #949494; background: #00d7d7"></span><span style="color: #949494; background: #949494">==========</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #949494">t</span><span style="color: #080808; background: #ffffff">ime</span><span style="color: #ffffff; background: #ffffff">   
</span><span style="color: #080808; background: #ffffff">i</span><span style="color: #ffffff; background: #ffffff">                                 </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">====</span><span style="color: #00d7d7; background: #949494"></span><span style="color: #00d7d7; background: #00d7d7">++++++</span><span style="color: #949494; background: #00d7d7"></span><span style="color: #949494; background: #949494">======</span><span style="color: #ff8787; background: #949494"></span><span style="color: #ff8787; background: #ff8787">===</span><span style="color: #949494; background: #ff8787"></span><span style="color: #949494; background: #949494">========</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">            </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #949494">D</span><span style="color: #080808; background: #ffffff">inner</span><span style="color: #ffffff; background: #ffffff"> 
</span><span style="color: #080808; background: #ffffff">p</span><span style="color: #ffffff; background: #ffffff">                           </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">===</span><span style="color: #00d7d7; background: #949494"></span><span style="color: #00d7d7; background: #00d7d7">+++++++</span><span style="color: #949494; background: #00d7d7"></span><span style="color: #949494; background: #949494">====</span><span style="color: #ff8787; background: #949494"></span><span style="color: #ff8787; background: #ff8787">=</span><span style="color: #949494; background: #ff8787"></span><span style="color: #949494; background: #949494">========</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #080808; background: #ffffff"></span><span style="color: #080808; background: #00d7d7">L</span><span style="color: #080808; background: #ffffff">unch</span><span style="color: #ffffff; background: #ffffff">  
       </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">==</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">==</span><span style="color: #00d7d7; background: #949494"></span><span style="color: #00d7d7; background: #00d7d7">+++++++</span><span style="color: #949494; background: #00d7d7"></span><span style="color: #949494; background: #949494">=====</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">                                    </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">=</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">      
  </span><span style="color: #4e4e4e; background: #ffffff">2.5</span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">==============</span><span style="color: #00d7d7; background: #949494"></span><span style="color: #00d7d7; background: #00d7d7">++++</span><span style="color: #949494; background: #00d7d7"></span><span style="color: #949494; background: #949494">====</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">                                                   
       </span><span style="color: #ff8787; background: #ffffff"></span><span style="color: #ff8787; background: #ff8787">======</span><span style="color: #00d7d7; background: #ff8787"></span><span style="color: #00d7d7; background: #00d7d7">++++++++</span><span style="color: #949494; background: #00d7d7"></span><span style="color: #949494; background: #949494">===</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">                                                        
       </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">==========</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">                                                               
       </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">=====</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">                                                                    
  </span><span style="color: #4e4e4e; background: #ffffff">0.0</span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #949494; background: #ffffff"></span><span style="color: #949494; background: #949494">==</span><span style="color: #ffffff; background: #949494"></span><span style="color: #ffffff; background: #ffffff">                                                                       
                </span><span style="color: #4e4e4e; background: #ffffff">10</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">20</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">30</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">40</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">50</span><span style="color: #ffffff; background: #ffffff">          
                                     </span><span style="color: #080808; background: #ffffff">bill</span><span style="color: #ffffff; background: #ffffff">                                       </span></pre>
<p><code>rush plot</code> cannot handle transparency, so a visual representation (see Figure @ref(fig:plot-trend-image) is much better in this case.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x bill --y tip --color time --geom smooth <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> plot-trend.pn
g
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">plot-trend.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/plot-trend.png" alt="Trend lines" width="90%" />
<p class="caption">
(#fig:plot-trend-image)Trend lines
</p>
</div>
<p>If you like to visualize the original points along with the trend lines, you need to resort to writing <code>ggplot2</code> code with <code>rush run</code> (see Figure @ref(fig:plot-trend-points-image)).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run --library ggplot2 <span style="color: #af8700">'ggplot(df, aes(x = bill, y = tip, color = time)) +
 geom_point() + geom_smooth()'</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> plot-trend-points.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">plot-trend-points.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/plot-trend-points.png" alt="Trend lines and original points combined" width="90%" />
<p class="caption">
(#fig:plot-trend-points-image)Trend lines and original points combined
</p>
</div>
</div>
<div id="creating-box-plots" class="section level3" number="7.4.9">
<h3 number="7.4.9"><span class="header-section-number">7.4.9</span> Creating Box Plots</h3>
<p>A box plot visualizes, for one or more features, a five-number summary: the minimum, the maximum, the sample median, and the first and third quartiles.
In this case we need to convert the <em>size</em> feature to a categorical one using the <code>factor()</code> function, otherwise all values of the <em>bill</em> feature will be lumped together.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x <span style="color: #af8700">'factor(size)'</span> --y bill --geom boxplot <span style="text-decoration: underline">tips.csv</span>                 
<span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #4e4e4e; background: #ffffff">50</span><span style="color: #ffffff; background: #ffffff">                               </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                            
                                   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       
                                   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       
  </span><span style="color: #4e4e4e; background: #ffffff">40</span><span style="color: #ffffff; background: #ffffff">                   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">            </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       
                                   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">   
                       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">   
</span><span style="color: #080808; background: #ffffff">b</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">30</span><span style="color: #ffffff; background: #ffffff">                               </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">   
</span><span style="color: #080808; background: #ffffff">i</span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                           
</span><span style="color: #080808; background: #ffffff">l</span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                           
</span><span style="color: #080808; background: #ffffff">l</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">20</span><span style="color: #ffffff; background: #ffffff">              </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                    
                  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                       
                  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                            
  </span><span style="color: #4e4e4e; background: #ffffff">10</span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                                        
      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                                                
                                                                                
          </span><span style="color: #4e4e4e; background: #ffffff">1</span><span style="color: #ffffff; background: #ffffff">            </span><span style="color: #4e4e4e; background: #ffffff">2</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">3</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">4</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">5</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">6</span><span style="color: #ffffff; background: #ffffff">        
                                   </span><span style="color: #080808; background: #ffffff">factor(size)</span><span style="color: #ffffff; background: #ffffff">                                 </span></pre>
<p>While the textual representation is not too bad, the visual one is much clearer (see Figure @ref(fig:plot-boxplot-image)).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x <span style="color: #af8700">'factor(size)'</span> --y bill --geom boxplot <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> plot-boxplot.p
ng
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">plot-boxplot.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/plot-boxplot.png" alt="A box plot" width="90%" />
<p class="caption">
(#fig:plot-boxplot-image)A box plot
</p>
</div>
<p>Unsurprisingly, this box plot shows that, on average, a larger party size leads to a higher bill.</p>
</div>
<div id="adding-labels" class="section level3" number="7.4.10">
<h3 number="7.4.10"><span class="header-section-number">7.4.10</span> Adding Labels</h3>
<p>The default labels are based on column names (or specification).
In the previous image, the label <em><code>factor(size)</code></em> should be improved.
Using the <code>--xlab</code> and <code>--ylab</code> options you can override the labels of the x and y axes.
A title can be added with the <code>--title</code> option.
Here’s a violin plot (which is a mashup of a box plot and a density plot) demonstrating this (see also Figure @ref(fig:plot-labels-image).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x <span style="color: #af8700">'factor(size)'</span> --y bill --geom violin --title <span style="color: #af8700">'Distribution of b
ill amount per party size'</span> --xlab <span style="color: #af8700">'Party size'</span> --ylab <span style="color: #af8700">'Bill (USD)'</span> <span style="text-decoration: underline">tips.csv</span>     
<span style="color: #ffffff"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #080808; background: #ffffff">Distribution</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #080808; background: #ffffff">of</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #080808; background: #ffffff">bill</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #080808; background: #ffffff">amount</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #080808; background: #ffffff">per</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #080808; background: #ffffff">party</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #080808; background: #ffffff">size</span><span style="color: #ffffff; background: #ffffff">                                   
  </span><span style="color: #4e4e4e; background: #ffffff">50</span><span style="color: #ffffff; background: #ffffff">                               </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                            
                                   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       
                                   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       
  </span><span style="color: #4e4e4e; background: #ffffff">40</span><span style="color: #ffffff; background: #ffffff">                   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       
</span><span style="color: #080808; background: #ffffff">B</span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">      
</span><span style="color: #080808; background: #ffffff">i</span><span style="color: #ffffff; background: #ffffff">                      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">     
</span><span style="color: #080808; background: #ffffff">l</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">30</span><span style="color: #ffffff; background: #ffffff">                  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">          </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">     
</span><span style="color: #080808; background: #ffffff">l</span><span style="color: #ffffff; background: #ffffff">                     </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">         </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                   
                     </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                 
</span><span style="color: #080808; background: #ffffff">(</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">20</span><span style="color: #ffffff; background: #ffffff">                </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                              
</span><span style="color: #080808; background: #ffffff">U</span><span style="color: #ffffff; background: #ffffff">                   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">      </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                          
</span><span style="color: #080808; background: #ffffff">S</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #4e4e4e; background: #ffffff">10</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">   </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">        </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                           
</span><span style="color: #080808; background: #ffffff">D</span><span style="color: #ffffff; background: #ffffff">    </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">  </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">     </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                                       
</span><span style="color: #080808; background: #ffffff">)</span><span style="color: #ffffff; background: #ffffff">       </span><span style="color: #3a3a3a; background: #ffffff"></span><span style="color: #3a3a3a; background: #3a3a3a">%%%%%%</span><span style="color: #ffffff; background: #3a3a3a"></span><span style="color: #ffffff; background: #ffffff">                                                                  
          </span><span style="color: #4e4e4e; background: #ffffff">1</span><span style="color: #ffffff; background: #ffffff">            </span><span style="color: #4e4e4e; background: #ffffff">2</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">3</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">4</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">5</span><span style="color: #ffffff; background: #ffffff">           </span><span style="color: #4e4e4e; background: #ffffff">6</span><span style="color: #ffffff; background: #ffffff">        
                                    </span><span style="color: #080808; background: #ffffff">Party</span><span style="color: #ffffff; background: #ffffff"> </span><span style="color: #080808; background: #ffffff">size</span><span style="color: #ffffff; background: #ffffff">                                  </span></pre>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x <span style="color: #af8700">'factor(size)'</span> --y bill --geom violin --title <span style="color: #af8700">'Distribution of b
ill amount per party size'</span> --xlab <span style="color: #af8700">'Party size'</span> --ylab <span style="color: #af8700">'Bill (USD)'</span> <span style="text-decoration: underline">tips.csv</span> <span style="color: #af8700">&gt;</span> pl
ot-labels.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">plot-labels.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/plot-labels.png" alt="A violin plot with a title and labels" width="90%" />
<p class="caption">
(#fig:plot-labels-image)A violin plot with a title and labels
</p>
</div>
<p>Annotating your visualization with proper labels and a title is especially useful if you want to share it with others (or your future self) so that it’s easier to understand what’s being shown.</p>
</div>
<div id="going-beyond-basic-plots" class="section level3" number="7.4.11">
<h3 number="7.4.11"><span class="header-section-number">7.4.11</span> Going Beyond Basic Plots</h3>
<p>Although <code>rush plot</code> is suitable for creating basic plots when you’re exploring data, it certainly has its limitations.
Sometimes you need more flexibility and sophisticated options such as multiple geometries, coordinate transformations, and theming.
In that case it might be worthwhile to learn more about the underlying package from which <code>rush plot</code> draws its capabilities, namely the <code>ggplot2</code> package for R.
When you’re more into Python than R, there’s the <a href="https://plotnine.readthedocs.io"><code>plotnine</code> package</a>, which is reimplementation of <code>ggplot2</code> for Python.</p>
</div>
</div>
<div id="summary-6" class="section level2" number="7.5">
<h2 number="7.5"><span class="header-section-number">7.5</span> Summary</h2>
<p>In this chapter we’ve looked at various ways to explore your data.
Both textual and graphical data visualizations have their pros and cons.
The graphical ones are obviously of much higher quality, but can be tricky to view at the command line.
This is where textual visualizations come in handy.
At least <code>rush</code> has, thanks to <code>R</code> and <code>ggplot2</code>, a consistent syntax for creating both types.</p>
<p>The next chapter is, once again, an intermezzo chapter in which I discuss how you can speed up your commands and pipelines.
Feel free to read that chapter later if you can’t wait to start modeling your data in <a href="#chapter-9-modeling-data">Chapter 9</a>.</p>
</div>
<div id="for-further-exploration-6" class="section level2" number="7.6">
<h2 number="7.6"><span class="header-section-number">7.6</span> For Further Exploration</h2>
<ul>
<li>A proper <code>ggplot2</code> tutorial is unfortunately beyond the scope of this book. If you want to get better at visualizing your data I strongly recommend that you invest some time in understanding the power and beauty of the grammar of graphics. Chapters 3 and 28 of the book <a href="https://r4ds.had.co.nz/"><em>R for Data Science</em></a> by Hadley Wickham and Garrett Grolemund are an excellent resource.</li>
<li>Speaking of Chapters 3 and 28, I <a href="https://datascienceworkshops.com/blog/plotnine-grammar-of-graphics-for-python/">translated those to Python using Plotnine and Pandas</a> in case you’re more into Python than R.</li>
</ul>
<!--chapter:end:07.Rmd-->
</div>
</div>
<div id="chapter-8-parallel-pipelines" class="section level1" number="8">
<h1 number="8"><span class="header-section-number">8</span> Parallel Pipelines</h1>
<!-- #TODO: SHOULD: Dicuss progress bar -->
<p>In the previous chapters, we’ve been dealing with commands and pipelines that take care of an entire task at once.
In practice, however, you may find yourself facing a task which requires the same command or pipeline to run multiple times.
For, example, you may need to:</p>
<ul>
<li>Scrape hundreds of web pages</li>
<li>Make dozens of API calls and transform their output</li>
<li>Train a classifier for a range of parameter values</li>
<li>Generate scatter plots for every pair of features in your dataset</li>
</ul>
<p>In any of the above examples, there’s a certain form of repetition involved.
With your favorite scripting or programming language, you take care of this with a for loop or a while loop.
On the command line, the first thing you might be inclined to do is to press <strong><code>Up</code></strong> to bring back the previous command, modify it if necessary, and press <strong><code>Enter</code></strong> to run the command again.
This is fine for two or three times, but imagine doing this dozens of times.
Such an approach quickly becomes cumbersome, inefficient, and prone to errors.
The good news is that you can write such loops on the command line as well.
That’s what this chapter is all about.</p>
<p>Sometimes, repeating a fast command one after the other (in a <em>serial</em> manner) is sufficient.
When you have multiple cores (and perhaps even multiple machines) it would be nice if you could make use of those, especially when you’re faced with a data-intensive task.
When using multiple cores or machines, the total running time may be reduced significantly.
In this chapter I will introduce a very powerful tool called <code>parallel</code><span class="citation"><a href="#fn95" class="footnote-ref" id="fnref95"><sup>95</sup></a></span> that can take care of exactly this. It enables you to apply a command or pipeline for a range of arguments such as numbers, lines, and files.
Plus, as the name implies, it allows you to run your commands in <em>parallel</em>.</p>
<div id="overview-5" class="section level2" number="8.1">
<h2 number="8.1"><span class="header-section-number">8.1</span> Overview</h2>
<p>This intermezzo chapter discusses several approaches to speed up tasks that require commands and pipelines to be run many times.
My main goal is to demonstrate to you the flexibility and power of <code>parallel</code>.
Because this tool can be combined with any other tool discussed in this book, it will positively change the way you use the command line for data science.
In this chapter, you’ll learn about:</p>
<ul>
<li>Running commands in serial to a range of numbers, lines, and files</li>
<li>Breaking a large task into several smaller tasks</li>
<li>Running pipelines in parallel</li>
<li>Distributing pipelines to multiple machines</li>
</ul>
<p>This chapter starts with the following files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch08</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 20K
-rw-r--r-- 1 dst dst  126 Dec 14 11:54 emails.txt
-rw-r--r-- 1 dst dst   61 Dec 14 11:54 movies.txt
-rwxr-xr-x 1 dst dst  125 Dec 14 11:54 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">slow.sh</span>*
-rw-r--r-- 1 dst dst 5.1K Dec 14 11:54 users.json</pre>
<p>The instructions to get these files are in <a href="#chapter-2-getting-started">Chapter 2</a>.
Any other files are either downloaded or generated using command-line tools.</p>
</div>
<div id="serial-processing" class="section level2" number="8.2">
<h2 number="8.2"><span class="header-section-number">8.2</span> Serial Processing</h2>
<p>Before I dive into parallelization, I’ll briefly discuss looping in a serial fashion.
It’s worthwhile to know how to do this because this functionality is always available, the syntax closely resembles looping in other programming languages, and it will really make you appreciate <code>parallel</code>.</p>
<p>From the examples provided in the introduction of this chapter, we can distill three types of items to loop over: numbers, lines, and files.
These three types of items will be discussed in the next three subsections, respectively.</p>
<div id="looping-over-numbers" class="section level3" number="8.2.1">
<h3 number="8.2.1"><span class="header-section-number">8.2.1</span> Looping Over Numbers</h3>
<p>Imagine that you need to compute the square of every even integer between 0 and 100. There’s a tool called <code>bc</code><span class="citation"><a href="#fn96" class="footnote-ref" id="fnref96"><sup>96</sup></a></span>, which is a <em>basic calculator</em> where you can pipe an equation to.
The command to compute the square of 4 looks as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">"4^2"</span> | <span style="color: #5f8700">bc</span>
16</pre>
<p>For a one-off calculation, this will do.
However, as mentioned in the introduction, you would need to be crazy to press <strong><code>Up</code></strong>, change the number, and press <strong><code>Enter</code></strong> 50 times!
In this case it’s better to let the shell do the hard work for you by using a for loop:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">for</span> i in {0..100..2}  <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">do</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">"</span><span style="color: #00afaf">$i</span><span style="color: #af8700">^2"</span> | <span style="color: #5f8700">bc</span>      <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10123;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">done</span> | <span style="color: #5f8700">trim</span>
0
4
16
36
64
100
144
196
256
324
… with 41 more lines</pre>
<p><span class="callout">&#10122;</span> The Z shell has a feature called brace expansion, which transforms <em><code>{0..100..2}</code></em> into a list separated by spaces: <em><code>0 2 4 … 98 100</code></em>. The variable <em><code>i</code></em> is assigned the value “0” in the first iteration, “1” in the second iteration, and so forth.
<br><span class="callout">&#10124;</span> The value of this variable can be used by prefixing it with a dollar sign (<em><code>$</code></em>). The shell will replace <em><code>$i</code></em> with its value before <code>echo</code> is being executed. Note that there can be more than one command between <em><code>do</code></em> and <em><code>done</code></em>.</p>
<p>Although the syntax may appear a bit odd compared to your favorite programming language, it’s worth remembering this because it’s always available in the shell.
I’ll introduce a better and more flexible way of repeating commands in a moment.</p>
</div>
<div id="looping-over-lines" class="section level3" number="8.2.2">
<h3 number="8.2.2"><span class="header-section-number">8.2.2</span> Looping Over Lines</h3>
<p>The second type of items you can loop over are lines.
These lines can come from either a file or from standard input.
This is a very generic approach because the lines can contain anything, including: numbers, dates, and email addresses.</p>
<p>Imagine that you’d want to send an email to all your contacts.
Let’s first generate some fake users using the free <a href="https://randomuser.me">Random User Generator API</a>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">curl</span> -s <span style="color: #af8700">"https://randomuser.me/api/1.2/?results=5&amp;seed=dsatcl2e"</span> <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">users.json</span>
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">users.json</span> <span style="color: #5f8700">jq</span> -r <span style="color: #af8700">'.results[].email'</span> <span style="color: #af8700">&gt;</span> emails
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">emails</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">emails</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ selma.andersen@example.com
   2   │ kent.clark@example.com
   3   │ ditmar.niehaus@example.com
   4   │ benjamin.robinson@example.com
   5   │ paulo.muller@example.com
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>You can loop over the lines from <em>emails</em> with a while loop:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">while</span> <span style="color: #5f8700">read</span> line                         <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">do</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">"Sending invitation to </span><span style="color: #00afaf">${line}</span><span style="color: #af8700">."</span>   <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10123;</span></span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">done</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">emails</span>                           <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10124;</span></span>
Sending invitation to selma.andersen@example.com.
Sending invitation to kent.clark@example.com.
Sending invitation to ditmar.niehaus@example.com.
Sending invitation to benjamin.robinson@example.com.
Sending invitation to paulo.muller@example.com.</pre>
<p><span class="callout">&#10122;</span> In this case you need to use a while loop because the Z shell does not know beforehand how many lines the input consists of.
<br><span class="callout">&#10123;</span> Although the curly braces around the <em>line</em> variable are not necessary in this case (since variable names cannot contain periods), it’s still good practice.
<br><span class="callout">&#10124;</span> This redirection can also be placed before <code>while</code>.</p>
<p>You can also provide input to a while loop interactively by specifying the special file standard input <em>/dev/stdin</em>. Press <strong><code>Ctrl-D</code></strong> when you are done.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">while</span> <span style="color: #5f8700">read</span> line; <span style="color: #af8700">do</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">"You typed: </span><span style="color: #00afaf">${line}</span><span style="color: #af8700">."</span>; <span style="color: #af8700">done</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">/dev/stdin</span>
one
You typed: one.
two
You typed: two.
three
You typed: three.</pre>
<p>This method, however, has the disadvantage that, once you press <strong><code>Enter</code></strong>, the commands between <em><code>do</code></em> and <em><code>done</code></em> are run immediately for that line of input. There’s no turning back.</p>
</div>
<div id="looping-over-files" class="section level3" number="8.2.3">
<h3 number="8.2.3"><span class="header-section-number">8.2.3</span> Looping Over Files</h3>
<p>In this section I discuss the third type of item that we often need to loop over: files.</p>
<p>To handle special characters, use <em>globbing</em> (i.e., pathname expansion) instead of <code>ls</code><span class="citation"><a href="#fn97" class="footnote-ref" id="fnref97"><sup>97</sup></a></span>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">for</span> chapter in /data/<span style="color: #0087ff">*</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">do</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">"Processing Chapter </span><span style="color: #00afaf">${chapter}</span><span style="color: #af8700">."</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">done</span>
Processing Chapter /data/ch02.
Processing Chapter /data/ch03.
Processing Chapter /data/ch04.
Processing Chapter /data/ch05.
Processing Chapter /data/ch06.
Processing Chapter /data/ch07.
Processing Chapter /data/ch08.
Processing Chapter /data/ch09.
Processing Chapter /data/ch10.
Processing Chapter /data/csvconf.</pre>
<p>Just as with brace expansion, the expression <em><code>/data/*</code></em> is first expanded into a list by the Z shell before it’s being processed by the for loop.</p>
<p>A more elaborate alternative to listing files is <code>find</code><span class="citation"><a href="#fn98" class="footnote-ref" id="fnref98"><sup>98</sup></a></span>, which:</p>
<ul>
<li>Can traverse down directories</li>
<li>Allows for elaborate searching on properties such as size, access time, and permissions</li>
<li>Handles special characters such as spaces and newlines</li>
</ul>
<p>For example, the following <code>find</code> invocation lists all files located under the directory <em>/data</em> that have <em>csv</em> as extension and are smaller than 2 kilobyte:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">find</span> <span style="text-decoration: underline">/data</span> -type f -name <span style="color: #af8700">'*.csv'</span> -size -2k
/data/ch03/tmnt-basic.csv
/data/ch03/tmnt-missing-newline.csv
/data/ch03/tmnt-with-header.csv
/data/ch05/irismeta.csv
/data/ch05/names-comma.csv
/data/ch05/names.csv
/data/ch07/datatypes.csv</pre>
</div>
</div>
<div id="parallel-processing" class="section level2" number="8.3">
<h2 number="8.3"><span class="header-section-number">8.3</span> Parallel Processing</h2>
<p>Let’s say that you have a very long running tool, such as the one shown here:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">slow.sh</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">slow.sh</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/bin/bash
</span>   2   │ <span style="color: #00afaf">echo</span> <span style="color: #5f8700">"Starting job $1"</span> <span style="color: #af005f">|</span> ts <span class="callout">&#10122;</span>
   3   │ duration<span style="color: #af005f">=</span><span style="color: #5f8700">$((</span><span style="color: #af8700">1</span><span style="color: #af005f">+</span><span style="color: #5f8700">RANDOM</span><span style="color: #af005f">%</span><span style="color: #af8700">5</span><span style="color: #5f8700">))</span> <span class="callout">&#10123;</span>
   4   │ sleep $duration <span class="callout">&#10124;</span>
   5   │ <span style="color: #00afaf">echo</span> <span style="color: #5f8700">"Job $1 took ${duration} seconds"</span> <span style="color: #af005f">|</span> ts
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p><span class="callout">&#10122;</span> <code>ts</code><span class="citation"><a href="#fn99" class="footnote-ref" id="fnref99"><sup>99</sup></a></span> adds a timestamp.
<br><span class="callout">&#10123;</span> The magic variable <em><code>RANDOM</code></em> calls an internal Bash function that returns a pseudorandom integer between 0 and 32767. Taking the remainder of the division of that integer by 5 and adding 1 ensures that <em>duration</em> is between 1 and 5.
<br><span class="callout">&#10124;</span> <code>sleep</code> pauses execution for a given number of seconds.</p>
<p>This process probably doesn’t take up all the available resources.
And it so happens that you need to run this command a lot of times.
For example, you need to download a whole sequence of files.</p>
<p>A naive way to parallelize is to run the commands in the background.
Let’s run <code>slow.sh</code> three times:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">for</span> i in {A..C}; <span style="color: #af8700">do</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">./slow.sh</span> $i &amp; <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">done</span>
[2] 386 <span class="callout">&#10123;</span>
[3] 388
[4] 391
 
<span style="font-weight: bold">$</span> Dec 14 11:54:18 Starting job A
Dec 14 11:54:18 Starting job B
Dec 14 11:54:18 Starting job C
Dec 14 11:54:20Dec 14 11:54:20 Job B took 2 seconds
 Job C took 2 seconds
 
[3]  - done       ./slow.sh $i
<span style="font-weight: bold">$</span>
[4]  + done       ./slow.sh $i
<span style="font-weight: bold">$</span> Dec 14 11:54:23 Job A took 5 seconds
 
[2]  + done       ./slow.sh $i
<span style="font-weight: bold">$</span></pre>
<p><span class="callout">&#10122;</span> The ampersand (<code>&amp;</code>) sends the command to the background, allowing the for loop to continue immediately with the next iteration.
<br><span class="callout">&#10123;</span> This line shows the job number given by the Z shell and the process ID, which can be used for more fine-grained job control. This topic, while powerful, is beyond the scope of this book.</p>

<div class="rmdnote">
Keep in mind that not everything can be parallelized.
API calls may be limited to a certain number, or some commands can only have one instance.
</div>
<p>Figure @ref(fig:diagram-parallel-processing) illustrates, on a conceptual level, the difference between serial processing, naive parallel processing, and parallel processing with GNU Parallel in terms of the number of concurrent processes and the total amount of time it takes to run everything.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0801.png" alt="Serial processing, naive parallel processing, and parallel processing with GNU Parallel" width="60%" />
<p class="caption">
(#fig:diagram-parallel-processing)Serial processing, naive parallel processing, and parallel processing with GNU Parallel
</p>
</div>
<p>There are two problems with this naive approach.
First, there’s no way to control how many processes you are running concurrently.
If you start too many jobs at once, they could be competing for the same resources such as CPU, memory, disk access, and network bandwidth.
This could lead to a longer time to run everything.
Second, it’s difficult to tell which output belongs to which input.
Let’s look at a better approach.</p>
<div id="introducing-gnu-parallel" class="section level3" number="8.3.1">
<h3 number="8.3.1"><span class="header-section-number">8.3.1</span> Introducing GNU Parallel</h3>
<p>Allow me to introduce <code>parallel</code>, a command-line tool that allows you to parallelize and distribute commands and pipelines.
The beauty of this tool is that existing tools can be used as they are; they do not need to be modified.</p>

<div class="rmdcaution">
Be aware that there are two command-line tools with the name <code>parallel</code>.
If you’re using the Docker image then you already have the correct one installed.
Otherwise, you can check that you have the correct one by running <code>parallel --version</code>.
It should say “GNU parallel.”
</div>
<p>Before I go into the details of <code>parallel</code>, here’s a little teaser to show you how easy it is to replace the for-loop from earlier:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 0 2 100 | <span style="color: #5f8700">parallel</span> <span style="color: #af8700">"echo {}^2 | bc"</span> | <span style="color: #5f8700">trim</span>
0
16
4
36
64
100
144
196
324
400
… with 41 more lines</pre>
<p>This is <code>parallel</code> in its simplest form: the items to loop over are passed via standard input and there aren’t any arguments other than the command that <code>parallel</code> needs to run.
See Figure @ref(fig:diagram-parallel-output) for an illustration of how <code>parallel</code> concurrently distributes input among processes and collects their outputs.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0802.png" alt="GNU Parallel concurrently distributes input among processes and collects their outputs" width="90%" />
<p class="caption">
(#fig:diagram-parallel-output)GNU Parallel concurrently distributes input among processes and collects their outputs
</p>
</div>
<p>As you can see it basically acts as a for loop.
Here’s another teaser, which replaces the for loop from the previous section.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">parallel</span> --jobs 2 <span style="text-decoration: underline">./slow.sh</span> ::: {A..C}
Dec 14 11:54:28 Starting job B
Dec 14 11:54:31 Job B took 3 seconds
Dec 14 11:54:28 Starting job A
Dec 14 11:54:33 Job A took 5 seconds
Dec 14 11:54:32 Starting job C
Dec 14 11:54:37 Job C took 5 seconds</pre>
<p>Here, using the <code>--jobs</code> option, I specify that <code>parallel</code> can run at most two jobs concurrently. The arguments to <code>slow.sh</code> are specified as an argument instead of via standard input.</p>
<p>With a whopping 159 different options, <code>parallel</code> offers a lot of functionality.
(Perhaps too much.)
Luckily you only need to know a handful in order to be effective.
The manual page is quite informative in case you need to use a less common option.</p>
</div>
<div id="specifying-input" class="section level3" number="8.3.2">
<h3 number="8.3.2"><span class="header-section-number">8.3.2</span> Specifying Input</h3>
<p>The most important argument to <code>parallel</code>, is the command or pipeline that you’d like to run for every input.
The question is: where should the input item be inserted in the command line?
If you don’t specify anything, then the input item will be appended to the end of the pipeline.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 3 | <span style="color: #5f8700">parallel</span> cowsay
 
 ___
&lt; 1 &gt;
 ---
        \   ^__^
         \  (oo)\_______
            (__)\       )\/\
                ||----w |
                ||     ||
 ___
&lt; 2 &gt;
 ---
        \   ^__^
         \  (oo)\_______
            (__)\       )\/\
                ||----w |
                ||     ||
 ___
&lt; 3 &gt;
 ---
        \   ^__^
         \  (oo)\_______
            (__)\       )\/\
                ||----w |
                ||     ||</pre>
<p>The above is the same as running:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cowsay</span> 1 <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">/dev/null</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #262626"><span class="callout">&#10122;</span></span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cowsay</span> 2 <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">/dev/null</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cowsay</span> 3 <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">/dev/null</span></pre>
<p><span class="callout">&#10122;</span> Because the output is the same as before, I redirect it to <em>/dev/null</em> to suppress it.</p>
<p>Although this often works, I advise you to be explicit about where the input item should be inserted in the command by using placeholders.
In this case, because you want to use the entire input line (a number) at once, you only need one placeholder.
You specify the placeholder, in other words, where to put the input item, with a pair of curly braces (<code>{}</code>):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 3 | <span style="color: #5f8700">parallel</span> cowsay {} <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">/dev/null</span></pre>

<div class="rmdnote">
There are other ways to provide input to <code>parallel</code>.
I prefer piping the input (as I do throughout this chapter) because that’s how most command-line tools are chained together into a pipeline.
The other ways involve syntax that’s not seen anywhere else.
Having said that, they do enable additional functionality, such as iterating over all possible combinations of multiple lists, so be sure to read <code>parallel</code>s manual page if you like to know more.
</div>
<p>When the input items are filenames, there are a couple of modifiers you can use only parts of the filename.
For example, with <code>{/}</code>, only the basename of the filename will be used:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">find</span> <span style="text-decoration: underline">/data/ch03</span> -type f | <span style="color: #5f8700">parallel</span> echo <span style="color: #af8700">'{#}\) \"{}\" has basename \"{/}\"'</span> <span class="callout">&#10122;</span>
1) "/data/ch03/tmnt-basic.csv" has basename "tmnt-basic.csv"
2) "/data/ch03/logs.tar.gz" has basename "logs.tar.gz"
3) "/data/ch03/tmnt-missing-newline.csv" has basename "tmnt-missing-newline.csv"
4) "/data/ch03/r-datasets.db" has basename "r-datasets.db"
5) "/data/ch03/top2000.xlsx" has basename "top2000.xlsx"
6) "/data/ch03/tmnt-with-header.csv" has basename "tmnt-with-header.csv"</pre>
<p><span class="callout">&#10122;</span> Characters such as parentheses (<code>)</code>) and quotes (<code>"</code>) have a special meaning in the shell. To use them literally you put a backslash <code>\</code> in front of them. This is called <em>escaping</em>.</p>
<p>If the input line has multiple parts separated by a delimiter you can add numbers to the placeholders.For example:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">input.csv</span> <span style="color: #5f8700">parallel</span> --colsep , <span style="color: #af8700">"mv {2} {1}"</span> <span style="color: #af8700">&gt;</span> <span style="text-decoration: underline">/dev/null</span></pre>
<p>Here, you can apply the same placeholder modifiers.
It is also possible to reuse the same input item.
If the input to <code>parallel</code> is a CSV file with a header, then you can use the column names as placeholders:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">input.csv</span> <span style="color: #5f8700">parallel</span> -C, --header : <span style="color: #af8700">"invite {name} {email}"</span></pre>

<div class="rmdtip">
If you ever wonder whether your placeholders are set up correctly, you can add the <code>--dryrun</code> option.
Instead of actually executing the command, <code>parallel</code> will print out all the commands exactly as if they would have been executed.
</div>
</div>
<div id="controlling-the-number-of-concurrent-jobs" class="section level3" number="8.3.3">
<h3 number="8.3.3"><span class="header-section-number">8.3.3</span> Controlling the Number of Concurrent Jobs</h3>
<p>By default, parallel runs one job per CPU core.
You can control the number of jobs that will be run concurrently with the <code>--jobs</code> or <code>-j</code> option.
Specifying a number means that many jobs will be run concurrently.
If you put a plus sign in front of the number then <code>parallel</code> will run <em>N</em> jobs plus the number of CPU cores. If you put a minus sign in front of the number then parallel will run <em>N-M</em> jobs.
Where <em>N</em> is the number of CPU cores.
You can also specify a percentage, where the default is 100% of the number of CPU cores.
The optimal number of jobs to run concurrently depends on the actual commands you are running.</p>
<!-- #TODO: Figure out where to type concurrently instead of parallel. And make sure all occurences of parallel are surrounded by backticks -->
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">parallel</span> -j0 <span style="color: #af8700">"echo Hi {}"</span>
Hi 1
Hi 2
Hi 3
Hi 4
Hi 5</pre>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">parallel</span> -j200% <span style="color: #af8700">"echo Hi {}"</span>
Hi 1
Hi 2
Hi 3
Hi 4
Hi 5</pre>
<p>If you specify <code>-j1</code>, then the commands will be run in serial. Even though this doesn’t do the name of the tool of justice, it still has its uses. For example, when you need to access an API which only allows one connection at a time. If you specify <code>-j0</code>, then parallel will run as many jobs in parallel as possible. This can be compared to your loop with the ampersand. This is not advised.</p>
</div>
<div id="logging-and-output" class="section level3" number="8.3.4">
<h3 number="8.3.4"><span class="header-section-number">8.3.4</span> Logging and Output</h3>
<p>To save the output of each command, you might be tempted to the following:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">parallel</span> <span style="color: #af8700">"echo </span><span style="color: #00afaf">\"</span><span style="color: #af8700">Hi {}</span><span style="color: #00afaf">\"</span><span style="color: #af8700"> &gt; hi-{}.txt"</span></pre>
<p>This will save the output into individual files.
Or, if you want to save everything into one big file you could do the following:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">parallel</span> <span style="color: #af8700">"echo Hi {}"</span> <span style="color: #af8700">&gt;&gt;</span> one-big-file.txt</pre>
<p>However, <code>parallel</code> offers the <code>--results</code> option, which stores the output in separate files.
For each job, <code>parallel</code> creates three files: <em>seq</em>, which holds the job number, <em>stdout</em> which contains the output produced by the job, and <em>stderr</em> which contains any errors produced by the job.
These three files are placed in subdirectories based on the input values.</p>
<p><code>parallel</code> still prints all the output, which is redundant in this case.
You can redirect both the standard input and standard output to <em>/dev/null</em> as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 10 | <span style="color: #5f8700">parallel</span> --results outdir <span style="color: #af8700">"curl 'https://anapioficeandfire.com/api/ch
aracters/{}' | jq -r '.aliases[0]'"</span> <span style="color: #af8700">2&gt;</span><span style="text-decoration: underline; color: #af8700"></span><span style="text-decoration: underline">/dev/null</span> <span style="color: #af8700">1&gt;&amp;</span>2
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">tree</span> <span style="text-decoration: underline">outdir</span> | <span style="color: #5f8700">trim
</span>outdir
└── 1
    ├── 1
    │   ├── seq
    │   ├── stderr
    │   └── stdout
    ├── 10
    │   ├── seq
    │   ├── stderr
    │   └── stdout
… with 34 more lines</pre>
<p>See Figure @ref(fig:diagram-parallel-results) for a pictorial overview of how the <code>--results</code> option works.</p>
<div class="figure" style="text-align: center">
<img src="images/dscl_0803.png" alt="GNU Parallel stores output in separate files with the `--results` option" width="90%" />
<p class="caption">
(#fig:diagram-parallel-results)GNU Parallel stores output in separate files with the <code>--results</code> option
</p>
</div>
<p>When you’re running multiple jobs in parallel, the order in which the jobs are run may not correspond to the order of the input.
The output of jobs is therefore also mixed up.
To keep the same order, specify the <code>--keep-order</code> option or <code>-k</code> option.</p>
<p>Sometimes it’s useful to record which input generated which output.
<code>parallel</code> allows you to <em>tag</em> the output with the <code>--tag</code> option, which prepends each line with the input item.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">parallel</span> --tag <span style="color: #af8700">"echo 'sqrt({})' | bc -l"</span>
1       1
3       1.73205080756887729352
4       2.00000000000000000000
2       1.41421356237309504880
5       2.23606797749978969640
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">parallel</span> --tag --keep-order <span style="color: #af8700">"echo '{1}*{2}' | bc -l"</span> ::: 3 4 ::: 5 6 7
3 5     15
3 6     18
3 7     21
4 5     20
4 6     24
4 7     28</pre>
</div>
<div id="creating-parallel-tools" class="section level3" number="8.3.5">
<h3 number="8.3.5"><span class="header-section-number">8.3.5</span> Creating Parallel Tools</h3>
<p>The <code>bc</code> tool, which I used in the beginning of the chapter, is not parallel by itself.
However, you can parallelize it using <code>parallel</code>.
The Docker image contains a tool called <code>pbc</code><span class="citation"><a href="#fn100" class="footnote-ref" id="fnref100"><sup>100</sup></a></span>.
Its code is shown here:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="color: #af005f">$(</span><span style="color: #5f8700">which</span> pbc<span style="color: #af005f">)</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">/usr/bin/dsutils/pbc</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/bin/bash
</span>   2   │ <span style="color: #5f8700"># pbc: parallel bc. First column of input CSV is mapped to {1}, second
</span>       │ <span style="color: #5f8700">to {2}, and so forth.</span>
   3   │ <span style="color: #5f8700">#</span>
   4   │ <span style="color: #5f8700"># Example usage: paste -d, &lt;(seq 100) &lt;(seq 100 -1 1) | ./pbc 'sqrt({1}
</span>       │ <span style="color: #5f8700">*{2})'</span>
   5   │ <span style="color: #5f8700">#</span>
   6   │ <span style="color: #5f8700"># Dependency: GNU parallel</span>
   7   │ <span style="color: #5f8700">#</span>
   8   │ <span style="color: #5f8700"># Author: http://jeroenjanssens.com</span>
   9   │
  10   │ parallel -C, -k -j100% <span style="color: #5f8700">"echo '$1' | bc -l"</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>This tool allows us to simplify the code used in the beginning of the chapter too.
And it can process comma-separated values simultaneously:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 100 | <span style="color: #5f8700">pbc</span> <span style="color: #af8700">'{1}^2'</span> | <span style="color: #5f8700">trim</span>
1
4
9
16
25
36
49
64
81
100
… with 90 more lines
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">paste</span> -d, <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">seq</span> 4<span style="color: #af005f">)</span> <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">seq</span> 4<span style="color: #af005f">)</span> <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">seq</span> 4<span style="color: #af005f">)</span> | <span style="color: #5f8700">pbc</span> <span style="color: #af8700">'sqrt({1}+{2})^{3}'</span>
1.41421356237309504880
4.00000000000000000000
14.69693845669906858905
63.99999999999999999969</pre>
</div>
</div>
<div id="distributed-processing" class="section level2" number="8.4">
<h2 number="8.4"><span class="header-section-number">8.4</span> Distributed Processing</h2>
<p>Sometimes you need more power than your local machine, even with all its cores, can offer.
Luckily, <code>parallel</code> can also leverage the power of remote machines, which really allows you to speed up your pipeline.</p>
<p>What’s great is that <code>parallel</code> doesn’t have to be installed on the remote machine.
All that’s required is that you can connect to the remote machine with the <em>Secure Shell</em> protocol (or SSH), which is also what <code>parallel</code> uses to distribute your pipeline.
(Having <code>parallel</code> installed is helpful because it can then determine how many cores to employ on each remote machine; more on this later.)</p>
<!-- #TODO: Introduce AWS EC2 -->
<!-- #TODO: COULD: Talk about Google Cloud and MS Azure -->
<p>First, I’m going to obtain a list of running AWS EC2 instances.
Don’t worry if you don’t have any remote machines, you can replace any occurrence of <code>--slf hostnames</code>, which tells <code>parallel</code> which remote machines to use, with <code>--sshlogin :</code>.
This way, you can still follow along with the examples in this section.</p>
<p>Once you know which remote machines to take over, we’re going to consider three flavors of distributed processing:</p>
<ul>
<li>Running ordinary commands on remote machines</li>
<li>Distributing local data directly among remote machines</li>
<li>Sending files to remote machines, process them, and retrieve the results</li>
</ul>
<div id="get-list-of-running-aws-ec2-instances" class="section level3" number="8.4.1">
<h3 number="8.4.1"><span class="header-section-number">8.4.1</span> Get List of Running AWS EC2 Instances</h3>
<!-- #TODO: Add links to GCP And Azure -->
<p>In this section we’re creating a file named <em>hostnames</em> that will contain one hostname of a remote machine per line.
I’m using Amazon Web Services (AWS) as an example.
I assume that you have an AWS account and that you know how to launch instances.
If you’re using a different cloud computing service (such as Google Cloud Platform or Microsoft Azure), or if you have your own servers, please make sure that you create a <em>hostnames</em> file yourself before continuing to the next section.</p>
<p>You can obtain a list of running AWS EC2 instances using <code>aws</code><span class="citation"><a href="#fn101" class="footnote-ref" id="fnref101"><sup>101</sup></a></span>, the command-line interface to the AWS API.
With <code>aws</code>, you can almost do everything you can do with the online AWS Management Console.</p>
<p>The command <code>aws ec2 describe-instances</code> returns a lot of information about all your EC2 instances in JSON format (see <a href="https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-instances.html">the online documentation</a> for more information).
You can extract the relevant fields using <code>jq</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">aws</span> ec2 describe-instances | <span style="color: #5f8700">jq</span> <span style="color: #af8700">'.Reservations[].Instances[] | {public_dns: .P
ublicDnsName, state: .State.Name}'</span></pre>
<p>The possible states of an EC2 instance are: <em><code>pending</code></em>, <em><code>running</code></em>, <em><code>shutting-down</code></em>, <em><code>terminated</code></em>, <em><code>stopping</code></em>, and <em><code>stopped</code></em>.
Because you can only distribute your pipeline to running instances, you filter out the non-running instances as follows:</p>
<pre><span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">aws</span> ec2 describe-instances | <span style="color: #5f8700">jq</span> -r <span style="color: #af8700">'.Reservations[].Instances[] | select(.Stat
e.Name=="running") | .PublicDnsName'</span> | <span style="color: #5f8700">tee</span> hostnames
ec2-54-88-122-140.compute-1.amazonaws.com
ec2-54-88-89-208.compute-1.amazonaws.com</pre>
<p>(Without the <code>-r</code> or <code>--raw-output</code> option, the hostnames would have been surrounded by double quotes.)
The output is saved to <em>hostnames</em>, so that I can pass this to <code>parallel</code> later.</p>
<p>As mentioned, <code>parallel</code> employs <code>ssh</code><span class="citation"><a href="#fn102" class="footnote-ref" id="fnref102"><sup>102</sup></a></span> to connect to the remote machines.
If you want to connect to your EC2 instances without typing the credentials every time, you can add something like the following text to the file <em>~/.ssh/config</em>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">~/.ssh/config</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">/home/dst/.ssh/config</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #af005f">Host</span> <span style="color: #af005f">*</span><span style="color: #00afaf">.amazonaws.com</span>
   2   │         <span style="color: #af005f">IdentityFile</span> <span style="color: #5f8700">~/.ssh/MyKeyFile.pem</span>
   3   │         <span style="color: #af005f">User</span> <span style="color: #5f8700">ubuntu</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>Depending on your which distribution your running, your user name may be different than <em><code>ubuntu</code></em>.</p>
</div>
<div id="running-commands-on-remote-machines" class="section level3" number="8.4.2">
<h3 number="8.4.2"><span class="header-section-number">8.4.2</span> Running Commands on Remote Machines</h3>
<p>The first flavor of distributed processing is to run ordinary commands on remote machines.
Let’s first double check that <code>parallel</code> is working by running the tool <code>hostname</code><span class="citation"><a href="#fn103" class="footnote-ref" id="fnref103"><sup>103</sup></a></span> on each EC2 instance:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">parallel</span> --nonall --sshloginfile hostnames hostname
ip-172-31-23-204
ip-172-31-23-205</pre>
<p>Here, the <code>--sshloginfile</code> or <code>--slf</code> option is used to refer to the file <em>hostnames</em>.
The <code>--nonall</code> option instructs <code>parallel</code> to execute the same command on every remote machine in the <em>hostnames</em> file without using any parameters.
Remember, if you don’t have any remote machines to utilize, you can replace <code>--slf hostnames</code> with <code>--sshlogin :</code> so that the command is run on your local machine:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">parallel</span> --nonall --sshlogin : hostname
data-science-toolbox</pre>
<p>Running the same command on every remote machine once only requires one core per machine. If you wanted to distribute the list of arguments passed in to <code>parallel</code> then it could potentially use more than one core. If the number of cores are not specified explicitly, <code>parallel</code> will try to determine this.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 2 | <span style="color: #5f8700">parallel</span> --slf hostnames echo <span style="color: #af8700">2&gt;&amp;</span>1
bash: parallel: command not found
parallel: Warning: Could not figure out number of cpus on ec2-54-88-122-140.comp
ute-1.amazonaws.com (). Using 1.
1
2</pre>
<p>In this case, I have <code>parallel</code> installed on one of the two remote machines.
I’m getting a warning message indicating that <code>parallel</code> is not found on one of them.
As a result, <code>parallel</code> cannot determine the number of cores and will default to using one core.
When you receive this warning message, you can do one of the following four things:</p>
<ul>
<li>Don’t worry, and be happy with using one core per machine</li>
<li>Specify the number of jobs for each machine via the <code>--jobs</code> or <code>-j</code> option</li>
<li>Specify the number of cores to use per machine by putting, for example, <em>2/</em> if you want two cores, in front of each hostname in the <em>hostnames</em> file</li>
<li>Install <code>parallel</code> using a package manager. For example, if the remote machines all run Ubuntu:</li>
</ul>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">parallel</span> --nonall --slf hostnames <span style="color: #af8700">"sudo apt-get install -y parallel"</span></pre>
</div>
<div id="distributing-local-data-among-remote-machines" class="section level3" number="8.4.3">
<h3 number="8.4.3"><span class="header-section-number">8.4.3</span> Distributing Local Data among Remote Machines</h3>
<p>The second flavor of distributed processing is to distribute local data directly among remote machines.
Imagine that you have one very large dataset that you want to process using multiple remote machines.
For simplicity, let’s sum all integers from 1 to 1000.
First, let’s double check that your input is actually being distributed by printing the hostname of the remote machine and the length of the input it received using <code>wc</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 1000 | <span style="color: #5f8700">parallel</span> -N100 --pipe --slf hostnames <span style="color: #af8700">"(hostname; wc -l) | paste -s
d:"</span>
ip-172-31-23-204:100
ip-172-31-23-205:100
ip-172-31-23-205:100
ip-172-31-23-204:100
ip-172-31-23-205:100
ip-172-31-23-204:100
ip-172-31-23-205:100
ip-172-31-23-204:100
ip-172-31-23-205:100
ip-172-31-23-204:100</pre>
<p>Excellent. You can see that your 1000 numbers get distributed evenly in subsets of 100 (as specified by <code>-N100</code>).
Now, you’re ready to sum all those numbers:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 1000 | <span style="color: #5f8700">parallel</span> -N100 --pipe --slf hostnames <span style="color: #af8700">"paste -sd+ | bc"</span> | <span style="color: #5f8700">paste</span> -sd
 
500500</pre>
<p>Here, you immediately also sum the ten sums you get back from the remote machines.
Let’s check that the answer is correct by doing the same calculation without <code>parallel</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 1000 | <span style="color: #5f8700">paste</span> -sd+ | <span style="color: #5f8700">bc</span>
500500</pre>
<p>Good, that works.
If you have a larger pipeline that you want to execute on the remote machines, you can also put it in a separate script and upload it with <code>parallel</code>.
I’ll demonstrate this by creating a very simple command-line tool called <code>add</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'#!/usr/bin/env bash'</span> <span style="color: #af8700">&gt;</span> add
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'paste -sd+ | bc'</span> <span style="color: #af8700">&gt;&gt;</span> <span style="text-decoration: underline">add</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">add</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">add</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env bash</span>
   2   │ paste -sd+ <span style="color: #af005f">|</span> bc
───────┴────────────────────────────────────────────────────────────────────────
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">chmod</span> u+x <span style="text-decoration: underline">add</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 1000 | <span style="color: #5f8700">./add</span>
500500</pre>
<p>Using the <code>--basefile</code> option, <code>parallel</code> first uploads the file <em>add</em> to all remote machines before running the jobs:</p>
<!-- #TODO: Explain --pipe -->
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 1000 |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">parallel</span> -N100 --basefile <span style="text-decoration: underline">add</span> --pipe --slf hostnames <span style="color: #af8700">'./add'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">./add</span>
500500</pre>
<p>Summing 1000 numbers is of course only a toy example.
Plus, it would’ve been much faster to do this locally.
Still, I hope it’s clear from this that <code>parallel</code> can be incredibly powerful.</p>
</div>
<div id="processing-files-on-remote-machines" class="section level3" number="8.4.4">
<h3 number="8.4.4"><span class="header-section-number">8.4.4</span> Processing Files on Remote Machines</h3>
<p>The third flavor of distributed processing is to send files to remote machines, process them, and retrieve the results.
Imagine that you want to count for each borough of New York City, how often they receive service calls on 311.
You don’t have that data on your local machine yet, so let’s first obtain it from the free <a href="https://data.cityofnewyork.us/">NYC Open Data API</a>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 0 100 900 | <span style="color: #5f8700">parallel</span>  <span style="color: #af8700">"curl -sL 'http://data.cityofnewyork.us/resource/erm
2-nwe9.json?</span><span style="color: #00afaf">\$</span><span style="color: #af8700">limit=100&amp;</span><span style="color: #00afaf">\$</span><span style="color: #af8700">offset={}' | jq -c '.[]' | gzip &gt; nyc-{#}.json.gz"</span></pre>
<p>You now have 10 files containing compressed JSON data:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span> nyc<span style="color: #0087ff">*</span>json.gz
-rw-r--r-- 1 dst dst 16K Dec 14 11:55 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-10.json.gz</span>
-rw-r--r-- 1 dst dst 13K Dec 14 11:56 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-1.json.gz</span>
-rw-r--r-- 1 dst dst 14K Dec 14 11:55 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-2.json.gz</span>
-rw-r--r-- 1 dst dst 14K Dec 14 11:55 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-3.json.gz</span>
-rw-r--r-- 1 dst dst 14K Dec 14 11:55 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-4.json.gz</span>
-rw-r--r-- 1 dst dst 16K Dec 14 11:55 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-5.json.gz</span>
-rw-r--r-- 1 dst dst 16K Dec 14 11:55 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-6.json.gz</span>
-rw-r--r-- 1 dst dst 16K Dec 14 11:55 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-7.json.gz</span>
-rw-r--r-- 1 dst dst 16K Dec 14 11:56 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-8.json.gz</span>
-rw-r--r-- 1 dst dst 15K Dec 14 11:56 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">nyc-9.json.gz</span></pre>
<p>Note that <code>jq -c '.[]'</code> is used to flatten the array of JSON objects so that there’s one object per line, with a total of 100 lines per file.
Using <code>zcat</code><span class="citation"><a href="#fn104" class="footnote-ref" id="fnref104"><sup>104</sup></a></span>, you directly print the contents of a compress file:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">zcat</span> <span style="text-decoration: underline">nyc-1.json.gz</span> | <span style="color: #5f8700">trim</span>
{"unique_key":"52779474","created_date":"2021-12-13T02:10:31.000","agency":"NYP…
{"unique_key":"52776058","created_date":"2021-12-13T02:09:50.000","agency":"NYP…
{"unique_key":"52775678","created_date":"2021-12-13T02:08:53.000","agency":"NYP…
{"unique_key":"52782776","created_date":"2021-12-13T02:07:37.000","closed_date"…
{"unique_key":"52778629","created_date":"2021-12-13T02:07:32.000","agency":"NYP…
{"unique_key":"52776019","created_date":"2021-12-13T02:07:23.000","agency":"NYP…
{"unique_key":"52776002","created_date":"2021-12-13T02:04:07.000","agency":"NYP…
{"unique_key":"52775975","created_date":"2021-12-13T02:02:46.000","agency":"NYP…
{"unique_key":"52776757","created_date":"2021-12-13T02:01:36.000","agency":"NYP…
{"unique_key":"52780492","created_date":"2021-12-13T02:01:35.000","agency":"NYP…
… with 90 more lines</pre>
<p>Let’s see what one line of JSON looks like using:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">zcat</span> <span style="text-decoration: underline">nyc-1.json.gz</span> | <span style="color: #5f8700">head</span> -n 1
{"unique_key":"52779474","created_date":"2021-12-13T02:10:31.000","agency":"NYPD
","agency_name":"New York City Police Department","complaint_type":"Encampment",
"descriptor":"N/A","location_type":"Subway","status":"In Progress","community_bo
ard":"Unspecified QUEENS","borough":"QUEENS","x_coordinate_state_plane":"1039396
","y_coordinate_state_plane":"195150","open_data_channel_type":"MOBILE","park_fa
cility_name":"Unspecified","park_borough":"QUEENS","bridge_highway_name":"E","br
idge_highway_segment":"Mezzanine","latitude":"40.702146602995356","longitude":"-
73.80111202259863","location":{"latitude":"40.702146602995356","longitude":"-73.
80111202259863","human_address":"{\"address\": \"\", \"city\": \"\", \"state\":
\"\", \"zip\": \"\"}"},":@computed_region_efsh_h5xi":"24340",":@computed_region_
f5dn_yrer":"41",":@computed_region_yeji_bk3q":"3",":@computed_region_92fq_4b7q":
"6",":@computed_region_sbqj_enih":"61"}</pre>
<p>If you were to get the total number of service calls per borough on your local machine, you would run the following command:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">zcat</span> nyc<span style="color: #0087ff">*</span>json.gz | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">jq</span> -r <span style="color: #af8700">'.borough'</span> | <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[A-Z] '</span> <span style="color: #af8700">'[a-z]_'</span> | <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sort</span> | <span style="color: #5f8700">uniq</span> -c | <span style="color: #5f8700">sort</span> -nr | <span class="callout">&#10125;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">awk</span> <span style="color: #af8700">'{print $2","$1}'</span> | <span class="callout">&#10126;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">header</span> -a borough,count | <span class="callout">&#10127;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvlook</span>
│ borough       │ count │
├───────────────┼───────┤
│ brooklyn      │   285 │
│ queens        │   271 │
│ manhattan     │   226 │
│ bronx         │   200 │
│ staten_island │    18 │</pre>
<p><span class="callout">&#10122;</span> Expand all compressed files using <code>zcat</code>.
<br><span class="callout">&#10123;</span> For each call, extract the name of the borough using <code>jq</code>.
<br><span class="callout">&#10124;</span> Convert borough names to lowercase and replace spaces with underscores (because <code>awk</code> splits on whitespace by default).
<br><span class="callout">&#10125;</span> Count the occurrences of each borough using <code>sort</code> and <code>uniq</code>.
<br><span class="callout">&#10126;</span> Reverse the two columns and delimit them by comma delimited using <code>awk</code>.
<br><span class="callout">&#10127;</span> Add a header using <code>header</code>.</p>
<p>Imagine, for a moment, that your own machine is so slow that you simply cannot perform this pipeline locally.
You can use <code>parallel</code> to distribute the local files among the remote machines, let them do the processing, and retrieve the results:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">ls</span> <span style="color: #0087ff">*</span>.json.gz | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">parallel</span> -v --basefile jq \ <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> --trc {.}.csv \ <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> --slf hostnames \ <span class="callout">&#10125;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">"zcat {} | ./jq -r '.borough' | tr '[A-Z] ' '[a-z]_' | sort | uniq -c | awk '{
print </span><span style="color: #00afaf">\$</span><span style="color: #af8700">2</span><span style="color: #00afaf">\"</span><span style="color: #af8700">,</span><span style="color: #00afaf">\"\$</span><span style="color: #af8700">1}' &gt; {.}.csv"</span> <span class="callout">&#10126;</span></pre>
<p><span class="callout">&#10122;</span> Print the list of files and pipe it into <code>parallel</code>
<br><span class="callout">&#10123;</span> Transmit the <code>jq</code> binary to each remote machine. Luckily, <code>jq</code> has no dependencies. This file will be removed from the remote machines afterwards because I specified the <code>--trc</code> option (which implies the <code>--cleanup</code> option). Note that the pipeline uses <code>./jq</code> instead of just <code>jq</code>. That’s because the pipeline needs to use the version which was uploaded and not the version that may or may not be on the search path.
<br><span class="callout">&#10124;</span> The command-line argument <code>--trc {.}.csv</code> is short for <code>--transfer --return {.}.csv --cleanup</code>. (The replacement string <em><code>{.}</code></em> gets replaced with the input filename without the last extension.) Here, this means that the JSON file gets transferred to the remote machine, the CSV file gets returned to the local machine, and both files will be removed after each job from the remote machine
<br><span class="callout">&#10125;</span> Specify a list of hostnames. Remember, if you want to try this out locally, you can specify <code>--sshlogin :</code> instead of <code>--slf hostnames</code>
<br><span class="callout">&#10126;</span> Note the escaping in the <code>awk</code> expression. Quoting can sometimes be tricky. Here, the dollar signs and the double quotes are escaped. If quoting ever gets too confusing, remember that you put the pipeline into a separate command-line tool just as I did with <code>add</code></p>
<p>If you, during this process, run <code>ls</code> on one of the remote machines, you would see that <code>parallel</code> indeed transfers (and cleans up) the binary <code>jq</code>, the JSON files, and CSV files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">ssh</span> <span style="color: #af005f">$(</span><span style="color: #5f8700">head</span> -n 1 hostnames<span style="color: #af005f">)</span> ls</pre>
<p>Each CSV file looks something like this:</p>
<pre><span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">nyc-1.json.csv</span>
bronx,3
brooklyn,5
manhattan,24
queens,3
staten_island,2</pre>
<p>You can sum the counts in each CSV file using <code>rush</code><span class="citation"><a href="#fn105" class="footnote-ref" id="fnref105"><sup>105</sup></a></span> and the tidyverse:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> nyc<span style="color: #0087ff">*</span>csv | <span style="color: #5f8700">header</span> -a borough,count |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">rush</span> run -t <span style="color: #af8700">'group_by(df, borough) %&gt;% summarize(count = sum(count))'</span> - |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvsort</span> -rc count | <span style="color: #5f8700">csvlook</span>
│ borough       │ count │
├───────────────┼───────┤
│ brooklyn      │   285 │
│ queens        │   271 │
│ manhattan     │   226 │
│ bronx         │   200 │
│ staten_island │    18 │</pre>
<p>Or, if you prefer to use SQL to aggregate results, you can use <code>csvsql</code> as discussed in <a href="#chapter-5-scrubbing-data">Chapter 5</a>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> nyc<span style="color: #0087ff">*</span>csv | <span style="color: #5f8700">header</span> -a borough,count |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvsql</span> --query <span style="color: #af8700">'SELECT borough, SUM(count) AS count FROM stdin GROUP BY boroug
h ORDER BY count DESC'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">csvlook</span>
│ borough       │ count │
├───────────────┼───────┤
│ brooklyn      │   285 │
│ queens        │   271 │
│ manhattan     │   226 │
│ bronx         │   200 │
│ staten_island │    18 │</pre>
</div>
</div>
<div id="summary-7" class="section level2" number="8.5">
<h2 number="8.5"><span class="header-section-number">8.5</span> Summary</h2>
<p>As a data scientist, you work with data–occasionally a lot of data.
This means that sometimes you need to run a command multiple times or distribute data-intensive commands over multiple cores.
In this chapter I have shown you how easy it is to parallelize commands.
<code>parallel</code> is a very powerful and flexible tool to speed up ordinary command-line tools and distribute them.
It offers a lot of functionality and in this chapter I’ve only been able to scratch the surface.
In the next chapter I’m going to cover the fourth step of the OSEMN model: modeling data.</p>
</div>
<div id="for-further-exploration-7" class="section level2" number="8.6">
<h2 number="8.6"><span class="header-section-number">8.6</span> For Further Exploration</h2>
<ul>
<li>Once you have a basic understanding of <code>parallel</code> and its most important options, I recommend that you take a look at <a href="https://www.gnu.org/software/parallel/parallel_tutorial.html">its online tutorial</a>. You’ll learn, among other things, how to specify different ways of specifying input, keep a log of all the jobs, and how to timeout, resume, and retry jobs. As creator of <code>parallel</code> Ole Tange in this tutorial says, “Your command line will love you for it.”</li>
</ul>
<!--chapter:end:08.Rmd-->
</div>
</div>
<div id="chapter-9-modeling-data" class="section level1" number="9">
<h1 number="9"><span class="header-section-number">9</span> Modeling Data</h1>
<p>In this chapter we’re going to perform the fourth step of the OSEMN model: modeling data.
Generally speaking, a model is an abstract or higher-level description of your data.
Modeling is a bit like creating visualizations in the sense that we’re taking a step back from the individual data points to see the bigger picture.</p>
<p>Visualizations are characterized by shapes, positions, and colors: we can interpret them by looking at them.
Models, on the other hand, are internally characterized by numbers, which means that computers can use them to do things like make predictions about a new data points.
(We can still visualize models so that we can try to understand them and see how they are performing.)</p>
<p>In this chapter I’ll consider three types of algorithms commonly used to model data:</p>
<ul>
<li>Dimensionality reduction</li>
<li>Regression</li>
<li>Classification</li>
</ul>
<p>These algorithms come from the field of statistics and machine learning, so I’m going to change the vocabulary a bit.
Let’s assume that I have a CSV file, also known as a <em>dataset</em>.
Each row, except for the header, is considered to be a <em>data point</em>.
Each data point has one or more <em>features</em>, or properties that have been measured.
Sometimes, a data point also has a <em>label</em>, which is, generally speaking, a judgment or outcome.
This becomes more concrete when I introduce the wine dataset below.</p>
<p>The first type of algorithm (dimensionality reduction) is most often unsupervised, which means that they create a model based on the features of the dataset only.
The last two types of algorithms (regression and classification) are by definition supervised algorithms, which means that they also incorporate the labels into the model.</p>

<div class="rmdcaution">
This chapter is by no means an introduction to machine learning.
That implies that I must skim over many details.
My general advice is that you become familiar with an algorithm before applying it to your data.
At the end of this chapter I recommend a few books about machine learning.
</div>
<div id="overview-6" class="section level2" number="9.1">
<h2 number="9.1"><span class="header-section-number">9.1</span> Overview</h2>
<p>In this chapter, you’ll learn how to:</p>
<ul>
<li>Reduce the dimensionality of your dataset using <code>tapkee</code><span class="citation"><a href="#fn106" class="footnote-ref" id="fnref106"><sup>106</sup></a></span>.</li>
<li>Predict the quality of white wine using <code>vw</code><span class="citation"><a href="#fn107" class="footnote-ref" id="fnref107"><sup>107</sup></a></span>.</li>
<li>Classify wine as red or white using <code>skll</code><span class="citation"><a href="#fn108" class="footnote-ref" id="fnref108"><sup>108</sup></a></span>.</li>
</ul>
<p>This chapter starts with the following file:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch09</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 4.0K
-rw-r--r-- 1 dst dst 503 Dec 14 11:57 classify.cfg</pre>
<p>The instructions to get these files are in <a href="#chapter-2-getting-started">Chapter 2</a>.
Any other files are either downloaded or generated using command-line tools.</p>
</div>
<div id="more-wine-please" class="section level2" number="9.2">
<h2 number="9.2"><span class="header-section-number">9.2</span> More Wine Please!</h2>
<p>Throughout this chapter, I’ll be using a dataset of wine tasters’ notes on red and white varieties of Portuguese wine called vinho verde.
Each data point represents a wine. Each wine is rated on 11 physicochemical properties: (1) fixed acidity, (2) volatile acidity, (3) citric acid, (4) residual sugar, (5) chlorides, (6) free sulfur dioxide, (7) total sulfur dioxide, (8) density, (9) pH, (10) sulphates, and (11) alcohol.
There is also an overall quality score between 0 (very bad) and 10 (excellent), which is the median of at least three evaluation by wine experts. More information about this dataset is available at the <a href="http://archive.ics.uci.edu/ml/datasets/Wine+Quality">UCI Machine Learning Repository</a>.</p>
<p>The dataset is split into two files: one for white wine and one for red wine.
The very first step is to obtain the two files using <code>curl</code> (and of course <code>parallel</code> because I haven’t got all day):</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">parallel</span> <span style="color: #af8700">"curl -sL http://archive.ics.uci.edu/ml/machine-learning-databases/wi
ne-quality/winequality-{}.csv &gt; wine-{}.csv"</span> ::: red white</pre>
<p>The triple colon is just another way to pass data to <code>parallel</code>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> /data/.cache/wine-<span style="color: #0087ff">*</span>.csv <span style="text-decoration: underline">.</span></pre>
<p>Let’s inspect both files and count the number of lines:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wine-red.csv</span> <span style="color: #5f8700">nl</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">fold</span> | <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">trim</span>
     1  "fixed acidity";"volatile acidity";"citric acid";"residual sugar";"chlor
ides";"free sulfur dioxide";"total sulfur dioxide";"density";"pH";"sulphates";"a
lcohol";"quality"
     2  7.4;0.7;0;1.9;0.076;11;34;0.9978;3.51;0.56;9.4;5
     3  7.8;0.88;0;2.6;0.098;25;67;0.9968;3.2;0.68;9.8;5
     4  7.8;0.76;0.04;2.3;0.092;15;54;0.997;3.26;0.65;9.8;5
     5  11.2;0.28;0.56;1.9;0.075;17;60;0.998;3.16;0.58;9.8;6
     6  7.4;0.7;0;1.9;0.076;11;34;0.9978;3.51;0.56;9.4;5
     7  7.4;0.66;0;1.8;0.075;13;40;0.9978;3.51;0.56;9.4;5
     8  7.9;0.6;0.06;1.6;0.069;15;59;0.9964;3.3;0.46;9.4;5
… with 1592 more lines
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wine-white.csv</span> <span style="color: #5f8700">nl</span> | <span style="color: #5f8700">fold</span> | <span style="color: #5f8700">trim</span>
     1  "fixed acidity";"volatile acidity";"citric acid";"residual sugar";"chlor
ides";"free sulfur dioxide";"total sulfur dioxide";"density";"pH";"sulphates";"a
lcohol";"quality"
     2  7;0.27;0.36;20.7;0.045;45;170;1.001;3;0.45;8.8;6
     3  6.3;0.3;0.34;1.6;0.049;14;132;0.994;3.3;0.49;9.5;6
     4  8.1;0.28;0.4;6.9;0.05;30;97;0.9951;3.26;0.44;10.1;6
     5  7.2;0.23;0.32;8.5;0.058;47;186;0.9956;3.19;0.4;9.9;6
     6  7.2;0.23;0.32;8.5;0.058;47;186;0.9956;3.19;0.4;9.9;6
     7  8.1;0.28;0.4;6.9;0.05;30;97;0.9951;3.26;0.44;10.1;6
     8  6.2;0.32;0.16;7;0.045;30;136;0.9949;3.18;0.47;9.6;6
… with 4891 more lines
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">wc</span> -l wine-{red,white}.csv
  1600 wine-red.csv
  4899 wine-white.csv
  6499 total</pre>
<p><span class="callout">&#10122;</span> For clarity I use <code>nl</code> to add line numbers.
<br><span class="callout">&#10123;</span> To see the entire header, I use <code>fold</code>.</p>
<p>At first sight this data appears to be quite clean.
Still, let’s scrub it so that it conforms more with what most command-line tools expect.
Specifically, I’ll:</p>
<ul>
<li>Convert the header to lowercase.</li>
<li>Replace the semi-colons with commas.</li>
<li>Replace spaces with underscores.</li>
<li>Remove unnecessary quotes.</li>
</ul>
<p>The tool <code>tr</code> can take care of all these things.
Let’s use a for loop this time—for old times’ sake—to process both files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">for</span> COLOR in red white; <span style="color: #af8700">do</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">&lt;</span> wine-$COLOR.csv <span style="color: #5f8700">tr</span> <span style="color: #af8700">'[A-Z]; '</span> <span style="color: #af8700">'[a-z],_'</span> | <span style="color: #5f8700">tr</span> -d \" <span style="color: #af8700">&gt;</span> wine-${COLOR}-clean.csv
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">done</span></pre>
<p>Let’s also create a single dataset by combining the two files.
I’ll use <code>csvstack</code><span class="citation"><a href="#fn109" class="footnote-ref" id="fnref109"><sup>109</sup></a></span> to add a column named <em>type</em>, which will be “red” for rows of the first file, and “white” for rows of the second file:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstack</span> -g red,white -n type wine-{red,white}-clean.csv | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">xsv</span> select 2-,1 <span style="color: #af8700">&gt;</span> wine.csv <span class="callout">&#10123;</span></pre>
<p><span class="callout">&#10122;</span> The new column <em>type</em> is placed at the beginning by <code>csvstack</code>.
<br><span class="callout">&#10123;</span> Some algorithms assume that the label is the last column, so I use <code>xsv</code> to move the column <em>type</em> to the end.</p>
<p>It’s good practice to check whether there are any missing values in this dataset, because most machine learning algorithms can’t handle them:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstat</span> <span style="text-decoration: underline">wine.csv</span> --nulls
  1. fixed_acidity: False
  2. volatile_acidity: False
  3. citric_acid: False
  4. residual_sugar: False
  5. chlorides: False
  6. free_sulfur_dioxide: False
  7. total_sulfur_dioxide: False
  8. density: False
  9. ph: False
 10. sulphates: False
 11. alcohol: False
 12. quality: False
 13. type: False</pre>
<p>Excellent!
If there were any missing values, we could fill them with, say, the average or most common value of that feature.
An alternative, less subtle approach is to remove any data points that have at least one missing value.
Just out of curiosity, let’s see what the distribution of quality looks like for both red and white wines.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run -t <span style="color: #af8700">'ggplot(df, aes(x = quality, fill = type)) + geom_density(adjust =
 3, alpha = 0.5)'</span> <span style="text-decoration: underline">wine.csv</span> <span style="color: #af8700">&gt;</span> wine-quality.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">wine-quality.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/wine-quality.png" alt="Comparing the quality of red and white wines using a density plot" width="90%" />
<p class="caption">
(#fig:plot_wine_quality)Comparing the quality of red and white wines using a density plot
</p>
</div>
<p>From the density plot you can see the quality of white wine is distributed more towards higher values.
Does this mean that white wines are overall better than red wines, or that the white wine experts more easily give higher scores than red wine experts?
That’s something that the data doesn’t tell us.
Or is there perhaps a relationship between alcohol and quality?
Let’s use <code>rush</code> to find out:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x alcohol --y quality --color type --geom smooth <span style="text-decoration: underline">wine.csv</span> <span style="color: #af8700">&gt;</span> wine-a
lcohol-vs-quality.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">wine-alcohol-vs-quality.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/wine-alcohol-vs-quality.png" alt="Relationship between the alcohol contents of wine and its quality" width="90%" />
<p class="caption">
(#fig:plot_wine_alchohol_vs_quality)Relationship between the alcohol contents of wine and its quality
</p>
</div>
<p>Eureka! Ahem, let’s carry on with some modeling, shall we?</p>
</div>
<div id="dimensionality-reduction-with-tapkee" class="section level2" number="9.3">
<h2 number="9.3"><span class="header-section-number">9.3</span> Dimensionality Reduction with Tapkee</h2>
<p>The goal of dimensionality reduction is to map high-dimensional data points onto a lower dimensional mapping.
The challenge is to keep similar data points close together on the lower-dimensional mapping.
As we’ve seen in the previous section, our wine dataset contains 13 features.
I’ll stick with two dimensions because that’s straightforward to visualize.</p>
<p>Dimensionality reduction is often regarded as part of exploration.
It’s useful for when there are too many features for plotting.
You could do a scatter-plot matrix, but that only shows you two features at a time.
It’s also useful as a pre-processing step for other machine learning algorithms.</p>
<p>Most dimensionality reduction algorithms are unsupervised.
This means that they don’t employ the labels of the data points in order to construct the lower-dimensional mapping.</p>
<p>In this section I’ll look at two techniques: PCA, which stands for Principal Components Analysis<span class="citation"><a href="#fn110" class="footnote-ref" id="fnref110"><sup>110</sup></a></span> and t-SNE, which stands for t-distributed Stochastic Neighbor Embedding<span class="citation"><a href="#fn111" class="footnote-ref" id="fnref111"><sup>111</sup></a></span>.</p>
<div id="introducing-tapkee" class="section level3" number="9.3.1">
<h3 number="9.3.1"><span class="header-section-number">9.3.1</span> Introducing Tapkee</h3>
<p>Tapkee is a C++ template library for dimensionality reduction<span class="citation"><a href="#fn112" class="footnote-ref" id="fnref112"><sup>112</sup></a></span>.
The library contains implementations of many dimensionality reduction algorithms, including:</p>
<ul>
<li>Locally Linear Embedding</li>
<li>Isomap</li>
<li>Multidimensional Scaling</li>
<li>PCA</li>
<li>t-SNE</li>
</ul>
<p>More information about these algorithms can be found on <a href="http://tapkee.lisitsyn.me/">Tapkee’s website</a>.
Although Tapkee is mainly a library that can be included in other applications, it also offers a command-line tool <code>tapkee</code>.
I’ll use this to perform dimensionality reduction on our wine dataset.</p>
</div>
<div id="linear-and-non-linear-mappings" class="section level3" number="9.3.2">
<h3 number="9.3.2"><span class="header-section-number">9.3.2</span> Linear and Non-linear Mappings</h3>
<p>First, I’ll scale the features using standardization such that each feature is equally important.
This generally leads to better results when applying machine learning algorithms.</p>
<p>To scale I use <code>rush</code> and the <code>tidyverse</code> package.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run --tidyverse --output wine-scaled.csv \
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">'select(df, -type) %&gt;%</span> <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">scale() %&gt;%</span> <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">as_tibble() %&gt;%</span> <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #af8700">mutate(type = df$type)'</span> <span style="text-decoration: underline">wine.csv</span> <span class="callout">&#10125;</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> <span style="text-decoration: underline">wine-scaled.csv</span>
│ fixed_acidity │ volatile_acidity │ citric_acid │ residual_sugar │ chlorides │…
├───────────────┼──────────────────┼─────────────┼────────────────┼───────────┤…
│        0.142… │           2.189… │     -2.193… │        -0.745… │    …
│        0.451… │           3.282… │     -2.193… │        -0.598… │    …
│        0.451… │           2.553… │     -1.917… │        -0.661… │    …
│        3.074… │          -0.362… │      1.661… │        -0.745… │    …
│        0.142… │           2.189… │     -2.193… │        -0.745… │    …
│        0.142… │           1.946… │     -2.193… │        -0.766… │    …
│        0.528… │           1.581… │     -1.780… │        -0.808… │    …
│        0.065… │           1.885… │     -2.193… │        -0.892… │    …
… with 6489 more lines</pre>
<p><span class="callout">&#10122;</span> I need to temporary remove the column <em><code>type</code></em> because <code>scale()</code> only works on numerical columns.
<br><span class="callout">&#10123;</span> The <code>scale()</code> function accepts a data frame, but returns a matrix.
<br><span class="callout">&#10124;</span> The function <code>as_tibble()</code> converts the matrix back to a data frame.
<br><span class="callout">&#10125;</span> Finally, I add back the <em><code>type</code></em> column.</p>
<p>Now we apply both dimensionality reduction techniques and visualize the mapping using <code>Rio-scatter</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">xsv</span> select <span style="color: #af8700">'!type'</span> <span style="text-decoration: underline">wine-scaled.csv</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">header</span> -d | <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tapkee</span> --method pca | <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tee</span> wine-pca.txt | <span style="color: #5f8700">trim</span>
-0.568882,3.34818
-1.19724,3.22835
-0.952507,3.23722
-1.60046,1.67243
-0.568882,3.34818
-0.556231,3.15199
-0.53894,2.28288
1.104,2.56479
0.231315,2.86763
-1.18363,1.81641
… with 6487 more lines</pre>
<p><span class="callout">&#10122;</span> Deselect the column <em><code>type</code></em>
<br><span class="callout">&#10123;</span> Remove the header
<br><span class="callout">&#10124;</span> Apply PCA</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wine-pca.txt</span> <span style="color: #5f8700">header</span> -a pc1,pc2 | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">paste</span> -d, - <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">xsv</span> select type <span style="text-decoration: underline">wine-scaled.csv</span><span style="color: #af005f"></span><span style="color: #af005f">)</span> | <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tee</span> wine-pca.csv | <span style="color: #5f8700">csvlook</span>
│      pc1 │     pc2 │ type  │
├──────────┼─────────┼───────┤
│  -0.569… │  3.348… │ red   │
│  -1.197… │  3.228… │ red   │
│  -0.953… │  3.237… │ red   │
│  -1.600… │  1.672… │ red   │
│  -0.569… │  3.348… │ red   │
│  -0.556… │  3.152… │ red   │
│  -0.539… │  2.283… │ red   │
│   1.104… │  2.565… │ red   │
… with 6489 more lines</pre>
<p><span class="callout">&#10122;</span> Add back the header with columns <em><code>pc1</code></em> and <em><code>pc2</code></em>
<br><span class="callout">&#10123;</span> Add back the column <em><code>type</code></em></p>
<p>Now we can create a scatter plot:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> plot --x pc1 --y pc2 --color type --shape type <span style="text-decoration: underline">wine-pca.csv</span> <span style="color: #af8700">&gt;</span> wine-pca.pn
g
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">wine-pca.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/wine-pca.png" alt="Linear dimensionality reduction with PCA" width="90%" />
<p class="caption">
(#fig:unnamed-chunk-10)Linear dimensionality reduction with PCA
</p>
</div>
<p>Let’s perform t-SNE with the same approach:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">xsv</span> select <span style="color: #af8700">'!type'</span> <span style="text-decoration: underline">wine-scaled.csv</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">header</span> -d | <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tapkee</span> --method t-sne | <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">header</span> -a x,y | <span class="callout">&#10125;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">paste</span> -d, - <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">xsv</span> select type <span style="text-decoration: underline">wine-scaled.csv</span><span style="color: #af005f"></span><span style="color: #af005f">)</span> | <span class="callout">&#10126;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">rush</span> plot --x x --y y --color type --shape type <span style="color: #af8700">&gt;</span> wine-tsne.png <span class="callout">&#10127;</span></pre>
<p><span class="callout">&#10122;</span> Deselect the column <em><code>type</code></em>
<br><span class="callout">&#10123;</span> Remove the header
<br><span class="callout">&#10124;</span> Apply t-SNE
<br><span class="callout">&#10125;</span> Add back the header with columns <em><code>x</code></em> and <em><code>y</code></em>
<br><span class="callout">&#10126;</span> Add back the column <em><code>type</code></em>
<br><span class="callout">&#10127;</span> Create a scatter plot</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">wine-tsne.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/wine-tsne.png" alt="Non-linear dimensionality reduction with t-SNE" width="90%" />
<p class="caption">
(#fig:unnamed-chunk-12)Non-linear dimensionality reduction with t-SNE
</p>
</div>
<p>We can see that t-SNE does a better job than PCA at separating the red and white wines based on their physicochemical properties.
These scatter plots verify that the dataset has a certain structure; there’s a relationship between the features and the labels.
Knowing this, I’m comfortable moving forward by applying supervised machine learning.
I’ll start with a regression task and then continue with a classification task.</p>
</div>
</div>
<div id="regression-with-vowpal-wabbit" class="section level2" number="9.4">
<h2 number="9.4"><span class="header-section-number">9.4</span> Regression with Vowpal Wabbit</h2>
<p>In this section, I’m going to create a model that predicts the quality of the white wine, based on their physicochemical properties.
Because the quality is a number between 0 and 10, we can consider this as a regression task.</p>
<p>For this I’ll be using Vowpal Wabbit, or <code>vw</code>.</p>
<div id="preparing-the-data" class="section level3" number="9.4.1">
<h3 number="9.4.1"><span class="header-section-number">9.4.1</span> Preparing the Data</h3>
<p>Instead of working with CSV, <code>vw</code> has its own data format.
The tool <code>csv2vw</code><span class="citation"><a href="#fn113" class="footnote-ref" id="fnref113"><sup>113</sup></a></span> can, as its name implies, convert CSV to this format.
The <code>--label</code> option is used to indicate which column contains the labels.
Let’s examine the result:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csv2vw</span> <span style="text-decoration: underline">wine-white-clean.csv</span> --label quality | <span style="color: #5f8700">trim</span>
6 | alcohol:8.8 chlorides:0.045 citric_acid:0.36 density:1.001 fixed_acidity:7 …
6 | alcohol:9.5 chlorides:0.049 citric_acid:0.34 density:0.994 fixed_acidity:6.…
6 | alcohol:10.1 chlorides:0.05 citric_acid:0.4 density:0.9951 fixed_acidity:8.…
6 | alcohol:9.9 chlorides:0.058 citric_acid:0.32 density:0.9956 fixed_acidity:7…
6 | alcohol:9.9 chlorides:0.058 citric_acid:0.32 density:0.9956 fixed_acidity:7…
6 | alcohol:10.1 chlorides:0.05 citric_acid:0.4 density:0.9951 fixed_acidity:8.…
6 | alcohol:9.6 chlorides:0.045 citric_acid:0.16 density:0.9949 fixed_acidity:6…
6 | alcohol:8.8 chlorides:0.045 citric_acid:0.36 density:1.001 fixed_acidity:7 …
6 | alcohol:9.5 chlorides:0.049 citric_acid:0.34 density:0.994 fixed_acidity:6.…
6 | alcohol:11 chlorides:0.044 citric_acid:0.43 density:0.9938 fixed_acidity:8.…
… with 4888 more lines</pre>
<p>In this format, each line is one data point.
The line starts with the label, followed by a pipe symbol and then feature name/value pairs separated by spaces.
While this format may seem overly verbose when compared to the CSV format, it does offer more flexibility such as weights, tags, namespaces, and a sparse feature representation.
With the wine dataset we don’t need this flexibility, but it might be useful when applying <code>vw</code> to more complicated problems.
This <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format">article</a> explains the <code>vw</code> format in more detail.</p>
<p>One we’ve created, or <em>trained</em> a regression model, it can be used to make predictions about new, unseen data points.
In other words, if we give the model a wine it hasn’t seen before, it can predict, or <em>test</em>, its quality.
To properly evaluate the accuracy of these predictions, we need to set aside some data that will not be used for training.
It’s common to use 80% of the complete dataset for training and the remaining 20% for testing.</p>
<p>I can do this by first splitting the complete dataset into five equal parts using <code>split</code><span class="citation"><a href="#fn114" class="footnote-ref" id="fnref114"><sup>114</sup></a></span>.
I verify the number of data points in each part using <code>wc</code>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">csv2vw</span> <span style="text-decoration: underline">wine-white-clean.csv</span> --label quality |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">shuf</span> | <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">split</span> -d -n r/5 - wine-part-
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">wc</span> -l wine-part-<span style="color: #0087ff">*</span>
   980 wine-part-00
   980 wine-part-01
   980 wine-part-02
   979 wine-part-03
   979 wine-part-04
  4898 total</pre>
<p><span class="callout">&#10122;</span> The tool <code>shuf</code><span class="citation"><a href="#fn115" class="footnote-ref" id="fnref115"><sup>115</sup></a></span> randomizes the dataset to ensure that both the training and the test have similar quality distribution.</p>
<p>Now I can use the first part (so 20%) for the testing set <em>wine-test.vw</em> and combine the four remaining parts (so 80%) into the training set <em>wine-train.vw</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">mv</span> <span style="text-decoration: underline">wine-part-00</span> wine-test.vw
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> wine-part-<span style="color: #0087ff">*</span> <span style="color: #af8700">&gt;</span> wine-train.vw
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">rm</span> wine-part-<span style="color: #0087ff">*</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">wc</span> -l wine-<span style="color: #0087ff">*</span>.vw
   980 wine-test.vw
  3918 wine-train.vw
  4898 total</pre>
<p>Now we’re ready to train a model using <code>vw</code>.</p>
</div>
<div id="training-the-model" class="section level3" number="9.4.2">
<h3 number="9.4.2"><span class="header-section-number">9.4.2</span> Training the Model</h3>
<p>The tool <code>vw</code> accepts many different options (nearly 400!).
Luckily, you don’t need all of them in order to be effective.
To annotate the options I use here, I’ll put each one on a separate line:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">vw</span> \
<span style="font-weight: bold">&gt;</span> --data <span style="text-decoration: underline">wine-train.vw</span> \ <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> --final_regressor wine.model \ <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> --passes 10 \ <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> --cache_file wine.cache \ <span class="callout">&#10125;</span>
<span style="font-weight: bold">&gt;</span> --nn 3 \ <span class="callout">&#10126;</span>
<span style="font-weight: bold">&gt;</span> --quadratic :: \ <span class="callout">&#10127;</span>
<span style="font-weight: bold">&gt;</span> --l2 0.000005 \ <span class="callout">&#10128;</span>
<span style="font-weight: bold">&gt;</span> --bit_precision 25 <span class="callout">&#10129;</span>
creating quadratic features for pairs: ::
WARNING: any duplicate namespace interactions will be removed
You can use --leave_duplicate_interactions to disable this behaviour.
using l2 regularization = 5e-06
final_regressor = wine.model
Num weight bits = 25
learning rate = 0.5
initial_t = 0
power_t = 0.5
decay_learning_rate = 1
creating cache_file = wine.cache
Reading datafile = wine-train.vw
num sources = 1
Enabled reductions: gd, generate_interactions, nn, scorer
average  since         example        example  current  current  current
loss     last          counter         weight    label  predict features
25.000000 25.000000            1            1.0   5.0000   0.0000       78
21.514251 18.028502            2            2.0   5.0000   0.7540       78
23.981016 26.447781            4            4.0   6.0000   1.5814       78
21.543597 19.106179            8            8.0   7.0000   2.1586       78
16.715053 11.886508           16           16.0   7.0000   2.8977       78
12.412012 8.108970           32           32.0   6.0000   3.8832       78
7.698827 2.985642           64           64.0   8.0000   4.8759       78
4.547053 1.395279          128          128.0   7.0000   5.7022       78
2.780491 1.013930          256          256.0   6.0000   5.9425       78
1.797196 0.813900          512          512.0   7.0000   5.9101       78
1.292476 0.787756         1024         1024.0   4.0000   5.8295       78
1.026469 0.760462         2048         2048.0   6.0000   5.9139       78
0.945076 0.945076         4096         4096.0   6.0000   6.1987       78 h
0.792362 0.639647         8192         8192.0   6.0000   6.2091       78 h
0.690935 0.589508        16384        16384.0   5.0000   5.5898       78 h
0.643649 0.596364        32768        32768.0   6.0000   6.1262       78 h
 
finished run
number of examples per pass = 3527
passes used = 10
weighted example sum = 35270.000000
weighted label sum = 206890.000000
average loss = 0.585270 h
best constant = 5.865891
total feature number = 2749380</pre>
<p><span class="callout">&#10122;</span> The file <em>wine-train.vw</em> is used to train the model.
<br><span class="callout">&#10123;</span> The model, or <em>regressor</em>, will be stored in the file <em>wine.model</em>.
<br><span class="callout">&#10124;</span> Number of training passes.
<br><span class="callout">&#10125;</span> Caching is needed when making multiple passes.
<br><span class="callout">&#10126;</span> Use a neural network with 3 hidden units.
<br><span class="callout">&#10127;</span> Create and use quadratic features, based on all input features. Any duplicates will be removed by <code>vw</code>.
<br><span class="callout">&#10128;</span> Use l2 regularization.
<br><span class="callout">&#10129;</span> Use 25 bits to store the features.</p>
<p>Now that I have trained a regression model, let’s use it to make predictions.</p>
</div>
<div id="testing-the-model" class="section level3" number="9.4.3">
<h3 number="9.4.3"><span class="header-section-number">9.4.3</span> Testing the Model</h3>
<p>The model is stored in the file <em>wine.model</em>.
To use that model to make predictions, I run <code>vw</code> again, but now with a different set of options:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">vw</span> \
<span style="font-weight: bold">&gt;</span> --data <span style="text-decoration: underline">wine-test.vw</span> \ <span class="callout">&#10122;</span>
<span style="font-weight: bold">&gt;</span> --initial_regressor <span style="text-decoration: underline">wine.model</span> \ <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> --testonly \ <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> --predictions predictions \ <span class="callout">&#10125;</span>
<span style="font-weight: bold">&gt;</span> --quiet <span class="callout">&#10126;</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">predictions</span> | <span style="color: #5f8700">trim</span>
6.702528
6.537283
5.633761
6.569905
5.934127
5.485150
5.768181
6.452881
4.978302
5.834136
… with 970 more lines</pre>
<p><span class="callout">&#10122;</span> The file <em>wine-test.vw</em> is used to test the model.
<br><span class="callout">&#10123;</span> Use the model stored in the file <em>wine.model</em>.
<br><span class="callout">&#10124;</span> Ignore label information and just test.
<br><span class="callout">&#10125;</span> The predictions are stored in a file called <em>predictions</em>.
<br><span class="callout">&#10126;</span> Don’t output diagnostics and progress updates.</p>
<p>Let’s use <code>paste</code> to combine the predictions in the file <em>predictions</em> with the true, or <em>observed</em>, values that are in the file <em>wine-test.vw</em>.
Using <code>awk</code>, I can compare the predicted values with the observed values and compute the mean absolute error (MAE).
The MAE tells us how far off <code>vw</code> is on average, when it comes to predicting the quality of a white wine.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">paste</span> -d, <span style="text-decoration: underline">predictions</span> <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">cut</span> -d <span style="color: #af8700">'|'</span> -f 1 <span style="text-decoration: underline">wine-test.vw</span><span style="color: #af005f"></span><span style="color: #af005f">)</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tee</span> results.csv |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">awk</span> -F, <span style="color: #af8700">'{E+=sqrt(($1-$2)^2)} END {print "MAE: " E/NR}'</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">cowsay</span> <span class="callout">&#10122;</span>
 _______________
&lt; MAE: 0.586385 &gt;
 ---------------
        \   ^__^
         \  (oo)\_______
            (__)\       )\/\
                ||----w |
                ||     ||</pre>
<p>So, the predictions are on average about 0.6 points off.
Let’s visualize the relationship between the observed values and the predicted values using <code>rush plot</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">results.csv</span> <span style="color: #5f8700">header</span> -a <span style="color: #af8700">"predicted,observed"</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">rush</span> plot --x observed --y predicted --geom jitter <span style="color: #af8700">&gt;</span> wine-regression.png
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">display</span> <span style="text-decoration: underline">wine-regression.png</span></pre>
<div class="figure" style="text-align: center">
<img src="images/wine-regression.png" alt="Regression with Vowpal Wabbit" width="90%" />
<p class="caption">
(#fig:unnamed-chunk-20)Regression with Vowpal Wabbit
</p>
</div>
<p>I can imagine that the options used to the train the model can be a bit overwhelming.
Let’s see how <code>vw</code> performs when I use all the default values:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">vw</span> -d <span style="text-decoration: underline">wine-train.vw</span> -f wine2.model --quiet <span class="callout">&#10122;</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">vw</span> -data <span style="text-decoration: underline">wine-test.vw</span> -i <span style="text-decoration: underline">wine2.model</span> -t -p <span style="text-decoration: underline">predictions</span> --quiet <span class="callout">&#10123;</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">paste</span> -d, <span style="text-decoration: underline">predictions</span> <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">cut</span> -d <span style="color: #af8700">'|'</span> -f 1 <span style="text-decoration: underline">wine-test.vw</span><span style="color: #af005f"></span><span style="color: #af005f">)</span> | <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">awk</span> -F, <span style="color: #af8700">'{E+=sqrt(($1-$2)^2)} END {print "MAE: " E/NR}'</span>
MAE: 0.61905</pre>
<p><span class="callout">&#10122;</span> Train a regression model
<br><span class="callout">&#10123;</span> Test the regression model
<br><span class="callout">&#10124;</span> Compute mean absolute error</p>
<p>Apparently, with the default values, the MAE is 0.04 higher, meaning that the predictions are slightly worse.</p>
<p>In this section, I’ve only been able to scratch the surface of what <code>vw</code> can do.
There’s reason why it accepts so many options.
Besides regression, it also supports, among other things, binary classification, multi-class classification, reinforcement learning, and Latent Dirichlet Allocation.
<a href="https://vowpalwabbit.org/">Its website</a> contains many tutorials and articles to learn more.</p>
</div>
</div>
<div id="classification-with-scikit-learn-laboratory" class="section level2" number="9.5">
<h2 number="9.5"><span class="header-section-number">9.5</span> Classification with SciKit-Learn Laboratory</h2>
<!-- TODO: Explain SKLL better -->
<p>In this section I’m going to train a classification model, or <em>classifier</em>, that predicts whether a wines is either red or white.
While we could use <code>vw</code> for this, I’d like to demonstrate another tool: SciKit-Learn Laboratory (SKLL).
As the name implies, it’s built on top of SciKit-Learn, a popular machine learning package for Python.
SKLL, itself a Python package, provides the <code>run_experiment</code> tool, which makes it possible to use SciKit-Learn from the command line.
Instead of <code>run_experiment</code>, I use the alias <code>skll</code> because I find it easier to remember as it corresponds to the package name:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">alias</span> skll=run_experiment
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">skll</span>
usage: run_experiment [-h] [-a NUM_FEATURES] [-A] [-k] [-l] [-m MACHINES]
                      [-q QUEUE] [-r] [-v] [--version]
                      config_file [config_file ...]
run_experiment: error: the following arguments are required: config_file</pre>
<div id="preparing-the-data-1" class="section level3" number="9.5.1">
<h3 number="9.5.1"><span class="header-section-number">9.5.1</span> Preparing the Data</h3>
<p><code>skll</code> expects the training and test dataset to have the same filenames, located in separate directories.
Because its predictions are not necessarily in the same order as the original dataset, I add a column, <em><code>id</code></em>, that contains a unique identifier so that I can match the predictions with the correct data points.
Let’s create a balanced dataset:</p>
<pre><span style="font-weight: bold">$</span> NUM_RED=<span style="color: #af8700">"</span><span style="color: #af005f">$(</span><span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wine-red-clean.csv</span> <span style="color: #5f8700">wc</span> -l<span style="color: #af005f">)</span><span style="color: #af8700">"</span> <span class="callout">&#10122;</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstack</span> -n type -g red,white \ <span class="callout">&#10123;</span>
<span style="font-weight: bold">&gt;</span> <span style="text-decoration: underline">wine-red-clean.csv</span> \
<span style="font-weight: bold">&gt;</span> <span style="color: #af005f">&lt;(</span><span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wine-white-clean.csv</span> <span style="color: #5f8700">body</span> shuf | <span style="color: #5f8700">head</span> -n $NUM_RED<span style="color: #af005f">)</span> |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">body</span> shuf |
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">nl</span> -s, -w1 -v0 | <span class="callout">&#10124;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">sed</span> <span style="color: #af8700">'1s/0,/id,/'</span> | <span class="callout">&#10125;</span>
<span style="font-weight: bold">&gt;</span> <span style="color: #5f8700">tee</span> wine-balanced.csv | <span style="color: #5f8700">csvlook</span>
│    id │ type  │ fixed_acidity │ volatile_acidity │ citric_acid │ residual_sug…
├───────┼───────┼───────────────┼──────────────────┼─────────────┼─────────────…
│     1 │ white │          7.30 │            0.300 │        0.42 │           7.…
│     2 │ white │          6.90 │            0.210 │        0.81 │           1.…
│     3 │ red   │          7.80 │            0.760 │        0.04 │           2.…
│     4 │ red   │          7.90 │            0.300 │        0.68 │           8.…
│     5 │ red   │          8.80 │            0.470 │        0.49 │           2.…
│     6 │ white │          6.40 │            0.150 │        0.29 │           1.…
│     7 │ white │          7.80 │            0.210 │        0.34 │          11.…
│     8 │ white │          7.00 │            0.130 │        0.37 │          12.…
… with 3190 more lines</pre>
<p><span class="callout">&#10122;</span> Store the number of red wines in variable <em><code>NUM_RED</code></em>.
<br><span class="callout">&#10123;</span> Combine all red wines with a random sample of white wines.
<br><span class="callout">&#10124;</span> Add “line numbers” using <code>nl</code> in front of each line.
<br><span class="callout">&#10125;</span> Replace the “0” on the first line with “id” so that it’s a proper column name.</p>
<p>Let’s split this balanced dataset into a training set and a test set:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">mkdir</span> -p {train,test}
 
<span style="font-weight: bold">$</span> HEADER=<span style="color: #af8700">"</span><span style="color: #af005f">$(</span><span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wine-balanced.csv</span> <span style="color: #5f8700">header</span><span style="color: #af005f">)</span><span style="color: #af8700">"
</span>
<span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">wine-balanced.csv</span> <span style="color: #5f8700">header</span> -d | <span style="color: #5f8700">shuf</span> | <span style="color: #5f8700">split</span> -d -n r/5 - wine-part-
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">wc</span> -l wine-part-<span style="color: #0087ff">*</span>
   640 wine-part-00
   640 wine-part-01
   640 wine-part-02
   639 wine-part-03
   639 wine-part-04
  3198 total
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="text-decoration: underline">wine-part-00</span> | <span style="color: #5f8700">header</span> -a $HEADER <span style="color: #af8700">&gt;</span> test/features.csv &amp;&amp; <span style="color: #5f8700">rm</span> <span style="text-decoration: underline">wine-part-00</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> wine-part-<span style="color: #0087ff">*</span> | <span style="color: #5f8700">header</span> -a $HEADER <span style="color: #af8700">&gt;</span> train/features.csv &amp;&amp; <span style="color: #5f8700">rm</span> wine-part-<span style="color: #0087ff">*</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">wc</span> -l t<span style="color: #0087ff">*</span>/features.csv
   641 test/features.csv
  2559 train/features.csv
  3200 total</pre>
<p>Now that I have a balanced training dataset and a balanced test dataset, I can continue with building a classifier.</p>
</div>
<div id="running-the-experiment" class="section level3" number="9.5.2">
<h3 number="9.5.2"><span class="header-section-number">9.5.2</span> Running the Experiment</h3>
<p>Training a classifier in <code>skll</code> is done by defining an experiment in a configuration file.
It consists of several sections that specify, for example, where to look for the datasets, which classifiers
Here’s the configuration file <em>classify.cfg</em> that I’ll use:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">classify.cfg</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">classify.cfg</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ [General]
   2   │ <span style="color: #af005f">experiment_name</span> = <span style="color: #5f8700">wine</span>
   3   │ <span style="color: #af005f">task</span> = <span style="color: #5f8700">evaluate</span>
   4   │
   5   │ [Input]
   6   │ <span style="color: #af005f">train_directory</span> = <span style="color: #5f8700">train</span>
   7   │ <span style="color: #af005f">test_directory</span> = <span style="color: #5f8700">test</span>
   8   │ <span style="color: #af005f">featuresets</span> = <span style="color: #5f8700">[["features"]]</span>
   9   │ <span style="color: #af005f">feature_scaling</span> = <span style="color: #5f8700">both</span>
  10   │ <span style="color: #af005f">label_col</span> = <span style="color: #5f8700">type</span>
  11   │ <span style="color: #af005f">id_col</span> = <span style="color: #5f8700">id
</span>  12   │ <span style="color: #af005f">shuffle</span> = <span style="color: #5f8700">true</span>
  13   │ <span style="color: #af005f">learners</span> = <span style="color: #5f8700">["KNeighborsClassifier", "LogisticRegression", "DecisionTree
</span>       │ <span style="color: #5f8700">Classifier", "RandomForestClassifier"]</span>
  14   │ <span style="color: #af005f">suffix</span> = <span style="color: #5f8700">.csv</span>
  15   │
  16   │ [Tuning]
  17   │ <span style="color: #af005f">grid_search</span> = <span style="color: #5f8700">false</span>
  18   │ <span style="color: #af005f">objectives</span> = <span style="color: #5f8700">["neg_mean_squared_error"]</span>
  19   │ <span style="color: #af005f">param_grids</span> = <span style="color: #5f8700">[{}, {}, {}, {}]</span>
  20   │
  21   │ [Output]
  22   │ <span style="color: #af005f">logs</span> = <span style="color: #5f8700">output</span>
  23   │ <span style="color: #af005f">results</span> = <span style="color: #5f8700">output</span>
  24   │ <span style="color: #af005f">predictions</span> = <span style="color: #5f8700">output</span>
  25   │ <span style="color: #af005f">models</span> = <span style="color: #5f8700">output</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p>I run the experiment using <code>skll</code>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">skll</span> -l <span style="text-decoration: underline">classify.cfg</span> <span style="color: #af8700">2&gt;</span><span style="text-decoration: underline; color: #af8700"></span><span style="text-decoration: underline">/dev/null</span></pre>
<p>The option<code>-l</code> specifies to run in local mode.
<code>skll</code> also offers the possibility to run experiments on clusters.
The time it takes to run an experiment depends on the complexity of the chosen algorithms and the size of the data.</p>
</div>
<div id="parsing-the-results" class="section level3" number="9.5.3">
<h3 number="9.5.3"><span class="header-section-number">9.5.3</span> Parsing the Results</h3>
<p>Once all classifiers have been trained and tested, the results can be found in the directory <em>output</em>:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">ls</span> -1 <span style="text-decoration: underline">output</span>
wine_features_DecisionTreeClassifier.log
wine_features_DecisionTreeClassifier.model
wine_features_DecisionTreeClassifier_predictions.tsv
wine_features_DecisionTreeClassifier.results
wine_features_DecisionTreeClassifier.results.json
wine_features_KNeighborsClassifier.log
wine_features_KNeighborsClassifier.model
wine_features_KNeighborsClassifier_predictions.tsv
wine_features_KNeighborsClassifier.results
wine_features_KNeighborsClassifier.results.json
wine_features_LogisticRegression.log
wine_features_LogisticRegression.model
wine_features_LogisticRegression_predictions.tsv
wine_features_LogisticRegression.results
wine_features_LogisticRegression.results.json
wine_features_RandomForestClassifier.log
wine_features_RandomForestClassifier.model
wine_features_RandomForestClassifier_predictions.tsv
wine_features_RandomForestClassifier.results
wine_features_RandomForestClassifier.results.json
wine.log
wine_summary.tsv</pre>
<p><code>skll</code> generates four files for each classifier: one log, two with results, and one with predictions.
I extract the algorithm names and sort them by their accuracies using the following SQL query:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #af8700">&lt;</span> <span style="text-decoration: underline">output/wine_summary.tsv</span> <span style="color: #5f8700">csvsql</span> --query <span style="color: #af8700">"SELECT learner_name, accuracy FROM s
tdin ORDER BY accuracy DESC"</span> | <span style="color: #5f8700">csvlook</span> -I
│ learner_name           │ accuracy  │
├────────────────────────┼───────────┤
│ RandomForestClassifier │ 0.9921875 │
│ LogisticRegression     │ 0.990625  │
│ KNeighborsClassifier   │ 0.9890625 │
│ DecisionTreeClassifier │ 0.984375  │</pre>
<p>The relevant column here is <em><code>accuracy</code></em>, which indicates the percentage of data points that are classified correctly.
From this we see that actually all algorithms are performing really well.
The RandomForestClassifier comes out as best performing algorithm, closely followed by KNeighborsClassifier.</p>
<p>Each JSON file contains a confusion matrix, giving you additional insight into the performance of each classifier.
A confusion matrix is a table where the columns refer to the true labels (red and white) and the rows refer to the predicted labels.
Higher numbers on the diagonal mean more correct predictions.
With <code>jq</code> I can print the name of each classifier and extract the associated confusion matrix:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">jq</span> -r <span style="color: #af8700">'.[] | "\(.learner_name):\n\(.result_table)\n"'</span> output/<span style="color: #0087ff">*</span>.json
DecisionTreeClassifier:
+-------+-------+---------+-------------+----------+-------------+
|       |   red |   white |   Precision |   Recall |   F-measure |
+=======+=======+=========+=============+==========+=============+
|   red | [313] |       7 |       0.991 |    0.978 |       0.984 |
+-------+-------+---------+-------------+----------+-------------+
| white |     3 |   [317] |       0.978 |    0.991 |       0.984 |
+-------+-------+---------+-------------+----------+-------------+
(row = reference; column = predicted)
 
KNeighborsClassifier:
+-------+-------+---------+-------------+----------+-------------+
|       |   red |   white |   Precision |   Recall |   F-measure |
+=======+=======+=========+=============+==========+=============+
|   red | [314] |       6 |       0.997 |    0.981 |       0.989 |
+-------+-------+---------+-------------+----------+-------------+
| white |     1 |   [319] |       0.982 |    0.997 |       0.989 |
+-------+-------+---------+-------------+----------+-------------+
(row = reference; column = predicted)
 
LogisticRegression:
+-------+-------+---------+-------------+----------+-------------+
|       |   red |   white |   Precision |   Recall |   F-measure |
+=======+=======+=========+=============+==========+=============+
|   red | [315] |       5 |       0.997 |    0.984 |       0.991 |
+-------+-------+---------+-------------+----------+-------------+
| white |     1 |   [319] |       0.985 |    0.997 |       0.991 |
+-------+-------+---------+-------------+----------+-------------+
(row = reference; column = predicted)
 
RandomForestClassifier:
+-------+-------+---------+-------------+----------+-------------+
|       |   red |   white |   Precision |   Recall |   F-measure |
+=======+=======+=========+=============+==========+=============+
|   red | [315] |       5 |       1.000 |    0.984 |       0.992 |
+-------+-------+---------+-------------+----------+-------------+
| white |     0 |   [320] |       0.985 |    1.000 |       0.992 |
+-------+-------+---------+-------------+----------+-------------+
(row = reference; column = predicted)
 </pre>
<p>A confusion matrix is especially helpful when you have more than two classes, so that you can see which kind of misclassifications happen, and when the cost of an incorrect classification is not the same for each class.</p>
<p>From a usage perspective, it’s interesting to consider that <code>vw</code> and <code>skll</code> take two different approaches.
<code>vw</code> uses command-line options, whereas <code>skll</code> requires a separate file.
Both approaches have their advantages and disadvantages.
While command-line options enable more ad-hoc usage, a configuration file is perhaps easier to reproduce.
Then again, as we’ve seen, invoking <code>vw</code> with any number of options can easily be placed in script or in a <em>Makefile</em>.
The opposite, making <code>skll</code> accept options such that it doesn’t need a configuration file, is less straightforward.</p>
</div>
</div>
<div id="summary-8" class="section level2" number="9.6">
<h2 number="9.6"><span class="header-section-number">9.6</span> Summary</h2>
<p>In this chapter we’ve looked at modeling data.
Through examples I dived into three different machine learning tasks namely dimensionality reduction which is unsupervised and regression and classification which are both supervised.
A proper machine learning tutorial is unfortunately beyond the scope of this book.
In the next section I have a couple of recommendations in case you want to learn more about machine learning.
This was the fourth and last step of the OSEMN model for data science that I’m covering in this book.
The next chapter is the last intermezzo chapter and will be about leveraging the command line elsewhere.</p>
</div>
<div id="for-further-exploration-8" class="section level2" number="9.7">
<h2 number="9.7"><span class="header-section-number">9.7</span> For Further Exploration</h2>
<ul>
<li>The book <em>Python Machine Learning</em> by Sebastian Raschka and Vahid Mirjalili offers a comprehensive overview of machine learning and how to apply it using Python.</li>
<li>The later chapters of <em>R for Everyone</em> by Jared Lander explain how to accomplish various machine learning tasks using R.</li>
<li>If you want to get a deeper understanding of machine learning, I highly recommend you pick up <em>Pattern Recognition and Machine Learning</em> by Christopher Bishop and <em>Information Theory, Inference, and Learning Algorithms</em> by David MacKay.</li>
<li>If you’re interested in learning more about the t-SNE algorithm, I recommend the original article about it: <em>Visualizing Data Using T-SNE</em> by Laurens van der Maaten and Geoffrey Hinton.</li>
</ul>
<!--chapter:end:09.Rmd-->
</div>
</div>
<div id="chapter-10-polyglot-data-science" class="section level1" number="10">
<h1 number="10"><span class="header-section-number">10</span> Polyglot Data Science</h1>
<p>A polyglot is someone who speaks multiple languages.
A polyglot data scientist, as I see it, is someone who uses multiple programming languages, tools, and techniques to obtain, scrub, explore, and model data.</p>
<p>The command line stimulates a polyglot approach.
The command line doesn’t care in which programming language a tool is written, as long as they adhere to the Unix philosophy.
We saw that very clearly in <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>, where we created command-line tools in Bash, Python, and R.
Moreover, we executed SQL queries directly on CSV files and executed R expressions from the command line.
In short, we have already been doing polyglot data science without fully realizing it!</p>
<p>In this chapter I’m going take this further by flipping it around.
I’m going to show you how to leverage the command line from various programming languages and environments.
Because let’s be honest, we’re not going to spend our entire data science careers at the command line.
As for me, when I’m analyzing some data I often use the RStudio IDE and when I’m implementing something, I often use Python.
I use whatever helps me to get the job done.</p>
<p>I find it comforting to know that the command line is often within arm’s reach, without having to switch to a different application.
It allows me to quickly run a command without switching to a separate application and break my workflow.
Examples are downloading files with <code>curl</code>, inspecting a piece of data with <code>head</code>, creating a backup with <code>git</code>, and compiling a website with <code>make</code>.
Generally speaking, tasks that normally require a lot of code or simply cannot be done at all without the command line.</p>
<div id="overview-7" class="section level2" number="10.1">
<h2 number="10.1"><span class="header-section-number">10.1</span> Overview</h2>
<p>In this chapter, you’ll learn how to:</p>
<ul>
<li>Run a terminal within JupyterLab and RStudio IDE</li>
<li>Interact with arbitrary command-line tools in Python and R</li>
<li>Transform data using shell commands in Apache Spark</li>
</ul>
<p>This chapter starts with the following files:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch10</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 180K
drwxr-xr-x 2 dst dst 4.0K Dec 14 12:03 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">__pycache__</span>/
-rw-r--r-- 1 dst dst 164K Dec 14 12:03 alice.txt
-rwxr--r-- 1 dst dst  408 Dec 14 12:03 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">count.py</span>*
-rw-r--r-- 1 dst dst  460 Dec 14 12:03 count.R
-rw-r--r-- 1 dst dst 1.7K Dec 14 12:03 Untitled1337.ipynb</pre>
<p>The instructions to get these files are in <a href="#chapter-2-getting-started">Chapter 2</a>.
Any other files are either downloaded or generated using command-line tools.</p>
</div>
<div id="jupyter" class="section level2" number="10.2">
<h2 number="10.2"><span class="header-section-number">10.2</span> Jupyter</h2>
<p>Project Jupyter is an open-source project, born out of the IPython Project in 2014 as it evolved to support interactive data science and scientific computing across all programming languages.
Jupyter supports over 40 programming languages, including Python, R, Julia, and Scala.
In this section I’ll focus on Python.</p>
<p>The project includes JupyterLab, Jupyter Notebook, and Jupyter Console.
I’ll start with Jupyter Console, as it is the most basic one to work with Python in an interactive way.
Here’s a Jupyter Console session illustrating a couple of ways to leverage the command line.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">jupyter</span> console
Jupyter console 6.4.0
 
Python 3.9.4 (default, Apr  4 2021, 19:38:44)
Type 'copyright', 'credits' or 'license' for more information
IPython 7.23.0 -- An enhanced Interactive Python. Type '?' for help.
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">1</span><span style="color: #008700"></span><span style="color: #008700">]: </span>! date <span class="callout">&#10122;</span>
Sun May  2 01:45:06 PM CEST 2021
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">2</span><span style="color: #008700"></span><span style="color: #008700">]: </span>! pip install --upgrade requests
Requirement already satisfied: requests in /home/dst/.local/lib/python3.9/site-p
ackages (2.25.1)
Collecting requests
  Using cached requests-2.25.1-py2.py3-none-any.whl (61 kB)
  Downloading requests-2.25.0-py2.py3-none-any.whl (61 kB)
     |████████████████████████████████| 61 kB 2.1 MB/s
Requirement already satisfied: urllib3&lt;1.27,&gt;=1.21.1 in /home/dst/.local/lib/pyt
hon3.9/site-packages (from requests) (1.26.4)
Requirement already satisfied: certifi&gt;=2017.4.17 in /home/dst/.local/lib/python
3.9/site-packages (from requests) (2020.12.5)
Requirement already satisfied: chardet&lt;5,&gt;=3.0.2 in /usr/lib/python3/dist-packag
es (from requests) (4.0.0)
Requirement already satisfied: idna&lt;3,&gt;=2.5 in /home/dst/.local/lib/python3.9/si
te-packages (from requests) (2.10)
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">3</span><span style="color: #008700"></span><span style="color: #008700">]: </span>! head alice.txt
﻿Project Gutenberg's Alice's Adventures in Wonderland, by Lewis Carroll
 
This eBook is for the use of anyone anywhere at no cost and with
almost no restrictions whatsoever.  You may copy it, give it away or
re-use it under the terms of the Project Gutenberg License included
with this eBook or online at www.gutenberg.org
 
 
Title: Alice's Adventures in Wonderland
 
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">4</span><span style="color: #008700"></span><span style="color: #008700">]: len</span>(<span style="color: #008700">open</span>(<span style="color: #af5f00">"alice.txt"</span>).read().strip().split(<span style="color: #af5f00">"</span><span style="font-weight: bold; color: #af5f00">\n</span><span style="color: #af5f00"></span><span style="color: #af5f00">"</span>)) <span class="callout">&#10123;</span>
<span style="color: #ff0000">Out[</span><span style="font-weight: bold; color: #ff0000">4</span><span style="color: #ff0000"></span><span style="color: #ff0000">]: </span>3735
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">5</span><span style="color: #008700"></span><span style="color: #008700">]: </span>total_lines = ! &lt; alice.txt wc -l
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">6</span><span style="color: #008700"></span><span style="color: #008700">]: </span>total_lines
<span style="color: #ff0000">Out[</span><span style="font-weight: bold; color: #ff0000">6</span><span style="color: #ff0000"></span><span style="color: #ff0000">]: </span>['3735']
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">7</span><span style="color: #008700"></span><span style="color: #008700">]: int</span>(total_lines[<span style="color: #008700">0</span>]) <span class="callout">&#10124;</span>
<span style="color: #ff0000">Out[</span><span style="font-weight: bold; color: #ff0000">7</span><span style="color: #ff0000"></span><span style="color: #ff0000">]: </span>3735
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">8</span><span style="color: #008700"></span><span style="color: #008700">]: </span>url = <span style="color: #af5f00">"https://www.gutenberg.org/files/11/old/11.txt"</span>
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">9</span><span style="color: #008700"></span><span style="color: #008700">]: </span><span style="font-weight: bold; color: #008700">import</span> <span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087d7">requests</span> <span class="callout">&#10125;</span>
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">10</span><span style="color: #008700"></span><span style="color: #008700">]: </span><span style="font-weight: bold; color: #008700">with</span> <span style="color: #008700">open</span>(<span style="color: #af5f00">"alice2.txt"</span>, <span style="color: #af5f00">"wb"</span>) <span style="font-weight: bold"></span><span style="font-weight: bold; color: #008700">as</span> f:
<span style="color: #008700">    ...: </span>    response = requests.get(url)
<span style="color: #008700">    ...: </span>    f.write(response.content)
<span style="color: #008700">    ...: </span>
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">11</span><span style="color: #008700"></span><span style="color: #008700">]: </span>! curl <span style="color: #af5f00">'</span><span style="font-weight: bold; color: #af5f00"></span><span style="font-weight: bold; color: #af5f87">{url}</span><span style="color: #af5f00"></span><span style="color: #af5f00">'</span> &gt; alice3.txt <span class="callout">&#10126;</span>
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  163k  100  163k    0     0   211k      0 --:--:-- --:--:-- --:--:--  211k
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">12</span><span style="color: #008700"></span><span style="color: #008700">]: </span>! ls alice*txt
alice2.txt  alice3.txt  alice.txt
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">13</span><span style="color: #008700"></span><span style="color: #008700">]: </span>! rm -v alice{<span style="color: #008700">2</span>,<span style="color: #008700">3</span>}.txt <span class="callout">&#10127;</span>
zsh:1: no matches found: alice(2, 3).txt
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">14</span><span style="color: #008700"></span><span style="color: #008700">]: </span>! rm -v alice{{<span style="color: #008700">2</span>,<span style="color: #008700">3</span>}}.txt
removed 'alice2.txt'
removed 'alice3.txt'
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">15</span><span style="color: #008700"></span><span style="color: #008700">]: </span>lower = [<span style="color: #af5f00">"foo"</span>, <span style="color: #af5f00">"bar"</span>, <span style="color: #af5f00">"baz"</span>]
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">16</span><span style="color: #008700"></span><span style="color: #008700">]: </span>upper = ! echo <span style="color: #af5f00">'{"</span><span style="font-weight: bold; color: #af5f00">\n</span><span style="color: #af5f00"></span><span style="color: #af5f00">".join(lower)}'</span> | tr <span style="color: #af5f00">'[a-z]'</span> <span style="color: #af5f00">'[A-Z]'</span> <span class="callout">&#10128;</span>
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">17</span><span style="color: #008700"></span><span style="color: #008700">]: </span>upper
<span style="color: #ff0000">Out[</span><span style="font-weight: bold; color: #ff0000">17</span><span style="color: #ff0000"></span><span style="color: #ff0000">]: </span>['FOO', 'BAR', 'BAZ']
 
<span style="color: #008700">In [</span><span style="font-weight: bold; color: #008700"></span><span style="font-weight: bold; color: #00ff00">18</span><span style="color: #008700"></span><span style="color: #008700">]: </span>exit
Shutting down kernel
 </pre>
<p><span class="callout">&#10122;</span> You can run arbitrary shell commands and pipelines such as <code>date</code> or <code>pip</code> to install a Python package.
<br><span class="callout">&#10123;</span> Compare this line of Pyton code to count the number of lines in <em>alice.txt</em> with the invocation of <code>wc</code> below it.
<br><span class="callout">&#10124;</span> Note that standard output is returned as a list of strings, so in order to use the value of <em>total_lines</em>, get the first item and cast it to an integer.
<br><span class="callout">&#10125;</span> Compare this cell and the next to download a file with the invocation of <code>curl</code> below it.
<br><span class="callout">&#10126;</span> You can use Python variables as part of the shell command by using curly braces.
<br><span class="callout">&#10127;</span> If you want to use literal curly braces, type them twice.
<br><span class="callout">&#10128;</span> Using a Python variable as standard input can be done, but gets quite tricky as you can see.</p>
<p>Jupyter Notebook is, in essence, a browser-based version of Jupyter Console.
It supports the same ways to leverage the command line, including the exclamation mark and bash magic.
The biggest difference is that a notebook cannot only contain code, but also marked-up text, equations, and data visualizations.
It’s very popular among data scientists for this reason.
Jupyter Notebook is a separate project and environment, but I’d like to use JupyterLab to work with notebooks, because it offers a more complete IDE.</p>
<p>Figure Figure @ref(fig:jupyterlab) is a screenshot of JupyterLab, showing the file explorer (left), a code editor (middle), a notebook (right), and a terminal (bottom). The latter three all show ways to leverage the command line.
The code is something I get back to in the next section.
This particular notebook is quite similar to the console session I just discussed.
The terminal offers a complete shell for you to run command line tools.
Be aware that there’s no interactivity possible between this terminal, the code, and the notebook.
So this terminal is not really different from having a separate terminal application open, but it’s still helpful when you’re working inside a Docker container or on a remote server.</p>
<div class="figure" style="text-align: center">
<img src="images/screenshot_jupyterlab.png" alt="JupyterLab with the file explorer, a code editor, a notebook, and a terminal" width="90%" />
<p class="caption">
(#fig:jupyterlab)JupyterLab with the file explorer, a code editor, a notebook, and a terminal
</p>
</div>

<div class="rmdcaution">
This notebook in the screenshot also contains a cell using so-called <code>%%bash</code> magic, which allows you to write multi-line Bash scripts.
Because it’s much more difficult to use Python variables, I don’t recommend this approach.
You’re better off creating a Bash script in a separate file, and then executing it by using the exclamation mark (<code>!</code>).
</div>
</div>
<div id="python" class="section level2" number="10.3">
<h2 number="10.3"><span class="header-section-number">10.3</span> Python</h2>
<p>The <code>subprocess</code> module allows you to run command-line tools from Python and connect to their standard input and output.
This module is recommended over the older <code>os.system()</code> function.
It’s not run in a shell by default, but it’s possible to change that with the <code>shell</code> argument to <code>run()</code> function.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">bat</span> <span style="text-decoration: underline">count.py</span>
───────┬────────────────────────────────────────────────────────────────────────
       │ File: <span style="font-weight: bold">count.py</span>
───────┼────────────────────────────────────────────────────────────────────────
   1   │ <span style="color: #5f8700">#!/usr/bin/env python</span>
   2   │
   3   │ <span style="color: #af005f">from</span> subprocess <span style="color: #af005f">import</span> run <span class="callout">&#10122;</span>
   4   │ <span style="color: #af005f">from</span> sys <span style="color: #af005f">import</span> argv
   5   │
   6   │ <span style="color: #af005f">if</span> __name__ <span style="color: #af005f">==</span> <span style="color: #5f8700">"__main__"</span>:
   7   │
   8   │     _, filename, pattern <span style="color: #af005f">=</span> argv
   9   │
  10   │     <span style="color: #af005f">with</span> <span style="color: #00afaf">open</span>(filename) <span style="color: #af005f">as</span> f: <span class="callout">&#10123;</span>
  11   │         alice <span style="color: #af005f">=</span> f.read()
  12   │
  13   │     words <span style="color: #af005f">=</span> <span style="color: #5f8700">"</span><span style="color: #00afaf">\n</span><span style="color: #5f8700">"</span>.join(alice.split()) <span class="callout">&#10124;</span>
  14   │
  15   │     grep <span style="color: #af005f">=</span> run([<span style="color: #5f8700">"grep"</span>, <span style="color: #5f8700">"-i"</span>, pattern], <span class="callout">&#10125;</span>
  16   │                input <span style="color: #af005f">=</span> words,
  17   │                capture_output<span style="color: #af005f">=</span><span style="color: #af8700">True</span>,
  18   │                text<span style="color: #af005f">=</span><span style="color: #af8700">True</span>)
  19   │
  20   │     <span style="color: #00afaf">print</span>(<span style="color: #00afaf">len</span>(grep.stdout.strip().split(<span style="color: #5f8700">"</span><span style="color: #00afaf">\n</span><span style="color: #5f8700">"</span>))) <span class="callout">&#10126;</span>
───────┴────────────────────────────────────────────────────────────────────────</pre>
<p><span class="callout">&#10122;</span> The recommended way to leverage the command line is to use the <code>run()</code> function of the <code>subprocess</code> module.
<br><span class="callout">&#10123;</span> Open the file <em>filename</em>
<br><span class="callout">&#10124;</span> Split the entire text into words
<br><span class="callout">&#10125;</span> Run the command-line tool <code>grep</code>, where <em>words</em> is passed as standard input.
<br><span class="callout">&#10126;</span> The standard output is available as one long string. Here, I split it on each newline character to count the number of occurrences of <em>pattern</em>.</p>
<p>This command-line tools is used as follows:</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">./count.py</span> <span style="text-decoration: underline">alice.txt</span> alice
403</pre>
<p>Notice that the first argument of the <code>run</code> call on line 15 is a list of strings, where the first item is the name of the command-line tool, and the remaining items are arguments.
This is different from passing a single string.
This is also means that you don’t have any other shell syntax available that would allow for things such as redirection and piping.</p>
</div>
<div id="r" class="section level2" number="10.4">
<h2 number="10.4"><span class="header-section-number">10.4</span> R</h2>
<p>In R, there are several ways to leverage the command line.</p>
<p>In the example below, I start an R session and count the number of occurrences of the string <em>alice</em> in the book <em>Alice’s Adventures in Wonderland</em> using the <code>system2()</code> function.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">R</span> --quiet
&gt; lines &lt;- readLines("alice.txt") <span class="callout">&#10122;</span>
&gt; head(lines)
[1] "Project Gutenberg's Alice's Adventures in Wonderland, by Lewis Carroll"
[2] ""
[3] "This eBook is for the use of anyone anywhere at no cost and with"
[4] "almost no restrictions whatsoever.  You may copy it, give it away or"
[5] "re-use it under the terms of the Project Gutenberg License included"
[6] "with this eBook or online at www.gutenberg.org"
&gt; words &lt;- unlist(strsplit(lines, " ")) <span class="callout">&#10123;</span>
&gt; head(words)
[1] "Project"     "Gutenberg's" "Alice's"     "Adventures"  "in"
[6] "Wonderland,"
&gt; alice &lt;- system2("grep", c("-i", "alice"), input = words, stdout = TRUE) <span class="callout">&#10124;</span>
&gt; head(alice)
[1] "Alice's" "Alice's" "ALICE'S" "ALICE'S" "Alice"   "Alice"
&gt; length(alice) <span class="callout">&#10125;</span></pre>
<p><span class="callout">&#10122;</span> Read in the file <em>alice.txt</em>
<br><span class="callout">&#10123;</span> Split the text into words
<br><span class="callout">&#10124;</span> Invoke the command-line tool <code>grep</code> to only keep the lines that match the string <em>alice</em>. The character vector <em>words</em> is passed as standard input.
<br><span class="callout">&#10125;</span> Count the number of elements in the character vector <em>alice</em></p>
<p>A disadvantage of <code>system2()</code> is that it first writes the character vector to a file before passing it as standard input to the command-line tool.
This can be problematic when dealing with a lot of data and a lot of invocations.</p>
<p>It’s better to use a named pipe, because then no data will be written to disk, which is much more efficient.
This can be done with <code>pipe()</code> and <code>fifo()</code> functions.
Thanks to Jim Hester for suggesting this.
The code below demonstrates this:</p>
<pre>&gt; out_con &lt;- fifo("out", "w+") <span class="callout">&#10122;</span>
&gt; in_con &lt;- pipe("grep b &gt; out") <span class="callout">&#10123;</span>
&gt; writeLines(c("foo", "bar"), in_con) <span class="callout">&#10124;</span>
&gt; readLines(out_con) <span class="callout">&#10125;</span>
[1] "bar"</pre>
<p><span class="callout">&#10122;</span> The function <code>fifo()</code> creates a special first-in-first-out file called <em>out</em>. This is just a reference to a pipe connection (like stdin and stdout are). No data is actually written to disk.
<br><span class="callout">&#10123;</span> The tool <code>grep</code> will only keep lines that contain a <em>b</em> and write them the named pipe <em>out</em> .
<br><span class="callout">&#10124;</span> Write two values to standard input of the shell command.
<br><span class="callout">&#10125;</span> Read the standard output produces by <code>grep</code> as a character vector.
<br><span class="callout">&#10126;</span> Clean up the connections and delete the special file.</p>
<!-- check out the processx package https://processx.r-lib.org/. experimental at the time of writing. but seems very promising to working with connections in a more robust manner. -->
<!-- # all four options are executed in a shell.  -->
<!-- # no stdin and no stdout: system -->
<!-- # stdin but no stdout: writeLines(pipe) -->
<!-- # stdout but no stdin: readLines(pipe) -->
<!-- readLines(pipe("")) -->
<!-- # both stdin and stdout: fifo, readLines, writeLines -->
<!-- first-in first-out special file, named pipe -->
<!-- When processes are -->
<!-- exchanging data via the FIFO, the kernel passes all data -->
<!-- internally without writing it to the filesystem. -->
<!-- ## Leveraging the Command Line Elsewhere -->
<!-- ### Clipboard -->
<!-- yank -->
<!-- pbcopy pbpaste -->
<!-- when formatting emailadresses -->
<p>Because this requires quite a bit of boilerplate code (creating connections, writing, reading, cleaning up), I have written a helper function <code>sh()</code>.
Using the pipe operator (<code>%&gt;%</code>) from the <code>magrittr</code> package I chain together multiple shell commands.</p>
<pre>&gt; library(magrittr)
&gt;
&gt; sh &lt;- function(.data, command) {
+   temp_file &lt;- tempfile()
+   out_con &lt;- fifo(temp_file, "w+")
+   in_con &lt;- pipe(paste0(command, " &gt; ", temp_file))
+   writeLines(as.character(.data), in_con)
+   result &lt;- readLines(out_con)
+   close(out_con)
+   close(in_con)
+   unlink(temp_file)
+   result
+ }
&gt;
&gt; lines &lt;- readLines("alice.txt")
&gt; words &lt;- unlist(strsplit(lines, " "))
&gt;
&gt; sh(words, "grep -i alice") %&gt;%
+   sh("wc -l") %&gt;%
+   sh("cowsay") %&gt;%
+   cli::cat_boxx()
┌──────────────────────────────────┐
│                                  │
│    _____                         │
│   &lt; 403 &gt;                        │
│    -----                         │
│           \   ^__^               │
│            \  (oo)\_______       │
│               (__)\       )\/\   │
│                   ||----w |      │
│                   ||     ||      │
│                                  │
└──────────────────────────────────┘
&gt;
&gt; q("no")</pre>
</div>
<div id="rstudio" class="section level2" number="10.5">
<h2 number="10.5"><span class="header-section-number">10.5</span> RStudio</h2>
<p>The RStudio IDE is arguably the most popular environment for working with R.
When you open RStudio, you will first see the console tab:</p>
<div class="figure" style="text-align: center">
<img src="images/screenshot_rstudio_console.png" alt="RStudio IDE with console tab open" width="90%" />
<p class="caption">
(#fig:rstudio-console)RStudio IDE with console tab open
</p>
</div>
<p>The terminal tab is right next to the console tab.
If offers a complete shell:</p>
<div class="figure" style="text-align: center">
<img src="images/screenshot_rstudio_terminal.png" alt="RStudio IDE with terminal tab open" width="90%" />
<p class="caption">
(#fig:rstudio-terminal)RStudio IDE with terminal tab open
</p>
</div>
<p>Note that, just as with JupyterLab, this terminal is not connected to the console or any R scripts.</p>
</div>
<div id="apache-spark" class="section level2" number="10.6">
<h2 number="10.6"><span class="header-section-number">10.6</span> Apache Spark</h2>
<p>Apache Spark is a cluster-computing framework.
It’s the 800-pound gorilla you turn to when it’s impossible to fit your data in memory.
Spark itself is written in Scala, but you can also interact with it from Python using <a href="https://spark.apache.org/docs/latest/api/python/index.html">PySpark</a> and from R using <a href="https://spark.apache.org/docs/latest/sparkr.html">SparkR</a> or <a href="https://spark.rstudio.com/">sparklyr</a>.</p>
<p>Data processing and machine learning pipelines are defined through a series of transformations and one final action.
One such transformation is the <code>pipe()</code> transformation, which allows you to run the entire dataset through a shell command such as a Bash or Perl script.
The items in the dataset are written to standard input and the standard output is returned as an RDD of strings.</p>
<p>In the session below, I start a Spark shell and again count the number of occurrences of <em>alice</em> in the book <em>Alice’s Adventures in Wonderland</em>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">spark-shell</span> --master local[6]
Spark context Web UI available at http://3d1bec8f2543:4040
Spark context available as 'sc' (master = local[6], app id = local-16193763).
Spark session available as 'spark'.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 3.1.1
      /_/
 
Using Scala version 2.12.10 (OpenJDK 64-Bit Server VM, Java 11.0.10)
Type in expressions to have them evaluated.
Type :help for more information.
 
<span style="color: #af005f">scala&gt; </span>val lines = sc.textFile("alice.txt") <span class="callout">&#10122;</span>
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">lines</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">org.apache.spark.rdd.RDD[String]</span> = alice.txt MapPartitionsRDD[1] at textF
ile at &lt;console&gt;:24
 
<span style="color: #af005f">scala&gt; </span>lines.first()
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">res0</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">String</span> = Project Gutenberg's Alice's Adventures in Wonderland, by Lewis Ca
rroll
 
<span style="color: #af005f">scala&gt; </span>val words = lines.flatMap(line =&gt; line.split(" ")) <span class="callout">&#10123;</span>
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">words</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">org.apache.spark.rdd.RDD[String]</span> = MapPartitionsRDD[2] at flatMap at &lt;con
sole&gt;:25
 
<span style="color: #af005f">scala&gt; </span>words.take(5)
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">res1</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">Array[String]</span> = Array(Project, Gutenberg's, Alice's, Adventures, in)
 
<span style="color: #af005f">scala&gt; </span>val alice = words.pipe("grep -i alice") <span class="callout">&#10124;</span>
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">alice</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">org.apache.spark.rdd.RDD[String]</span> = PipedRDD[3] at pipe at &lt;console&gt;:25
 
<span style="color: #af005f">scala&gt; </span>alice.take(5)
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">res2</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">Array[String]</span> = Array(Alice's, Alice's, ALICE'S, ALICE'S, Alice)
 
<span style="color: #af005f">scala&gt; </span>val counts = alice.pipe("wc -l") <span class="callout">&#10125;</span>
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">counts</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">org.apache.spark.rdd.RDD[String]</span> = PipedRDD[4] at pipe at &lt;console&gt;:25
 
<span style="color: #af005f">scala&gt; </span>counts.collect()
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">res3</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">Array[String]</span> = Array(64, 72, 94, 93, 67, 13) <span class="callout">&#10126;</span>
 
<span style="color: #af005f">scala&gt; </span>counts.map(x =&gt; x.toInt).reduce(_ + _) <span class="callout">&#10127;</span>
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">res4</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">Int</span> = 403
 
<span style="color: #af005f">scala&gt; </span>sc.textFile("alice.txt").flatMap(line =&gt; line.split(" ")).pipe("grep -i a
lice").pipe("wc -l").map(x =&gt; x.toInt).reduce(_ + _)
<span style="font-weight: bold"></span><span style="font-weight: bold; color: #0087ff">res5</span>: <span style="font-weight: bold"></span><span style="font-weight: bold; color: #5f8700">Int</span> = 403 <span class="callout">&#10128;</span>
 </pre>
<p><span class="callout">&#10122;</span> Read <em>alice.txt</em> such that each line is an element.
<br><span class="callout">&#10123;</span> Split each element on spaces. In other words, each line is split into words.
<br><span class="callout">&#10124;</span> Pipe each partition through <code>grep</code> to keep only the elements that match the string <em>alice</em>.
<br><span class="callout">&#10125;</span> Pipe each partition through <code>wc</code> to count the number of elements.
<br><span class="callout">&#10126;</span> There’s one count for each partition.
<br><span class="callout">&#10127;</span> Sum all counts to get a final count. Note that elements first need to be converted from strings to integers.
<br><span class="callout">&#10128;</span> The above steps combined into a single command.</p>

<div class="rmdtip">
The <code>pipe()</code> transformation is also available in PySpark, SparkR, and sparklyr.
</div>
<!-- https://stackoverflow.com/questions/54239583/question-about-rdd-pipe-operator-on-apache-spark -->
<p>If you want to use a custom command-line tool in your pipeline, then you need to make sure that it’s present on all nodes in the cluster (known as the executors).
One way to do this is to specify the filename(s) with the <code>--files</code> option when you’re submitting Spark applications using <code>spark-submit</code>.</p>
<p>Matei Zaharia and Bill Chambers (the original author of Apache Spark) mention in their book <em>Spark: The Definitive Guide</em> that “[t]he <code>pipe</code> method is probably one of Spark’s more interesting methods.”
That’s quite the compliment!
I think it’s fantastic that the developers of Apache Spark added the ability to leverage a 50-year old technology.</p>
<!-- ### Notable mentions -->
<!-- - Julia: Blog post with an introduction: https://blog.leahhanson.us/post/julia/julia-commands.html -->
<!-- - Visual Studio Code https://code.visualstudio.com/docs/editor/integrated-terminal -->
<!-- - Emacs -->
<!-- - VIM (using ! command) -->
<!-- - OS: Guake, -->
<!-- - OS: iTerm2:    https://www.sharmaprakash.com.np/guake-like-dropdown-terminal-in-mac/ -->
<!-- https://github.com/shelljs/shelljs -->
<!-- https://amoffat.github.io/sh/ -->
<!-- https://plumbum.readthedocs.io/en/latest/ -->
<!-- <\!-- ## Other Combinations -\-> -->
<!-- - reticulate -->
<!-- - Rpy2 -->
<!-- - sparkr -->
<!-- - sparklyr -->
<!-- TODO: MUST: Write Summary or Conclusion -->
<!-- # Summary -->
<!-- TODO: MUST: Talk about other combinations between languages. This is already possible with this approah, but there are tighter integrations. -->
</div>
<div id="summary-9" class="section level2" number="10.7">
<h2 number="10.7"><span class="header-section-number">10.7</span> Summary</h2>
<p>In this chapter you learned several ways to leverage the command line in other situations, including programming languages and other environments.
It’s important to realize that the command line doesn’t exist in a vacuum.
What matters most is that you use tools, sometimes in combination, that reliably get the job done.</p>
<p>Now that we’ve had all the four OSEMN chapters and the four intermezzo chapters, it’s time to wrap this up and conclude in the final chapter.</p>
</div>
<div id="for-further-exploration-9" class="section level2" number="10.8">
<h2 number="10.8"><span class="header-section-number">10.8</span> For Further Exploration</h2>
<ul>
<li>There are also ways to integrating two programming languages directly, without the use of the command line. For example the <a href="https://rstudio.github.io/reticulate/"><code>reticulate</code> package</a> in R allows you to interface with Python directly.</li>
</ul>
<!--chapter:end:10.Rmd-->
</div>
</div>
<div id="chapter-11-conclusion" class="section level1" number="11">
<h1 number="11"><span class="header-section-number">11</span> Conclusion</h1>
<p>In this final chapter, the book comes to a close.
I’ll first recap what I’ve discussed in the previous ten chapters, and will then offer you three pieces of advice and provide some resources to further explore the related topics we touched upon.
Finally, in case you have any questions, comments, or new command-line tools to share, I provide a few ways to get in touch with me.</p>
<div id="lets-recap" class="section level2" number="11.1">
<h2 number="11.1"><span class="header-section-number">11.1</span> Let’s Recap</h2>
<p>This book explored the power of using the command line to do data science.
I find it an interesting observation that the challenges posed by this relatively young field can be tackled by such a time-tested technology.
I hope that you now see what the command line is capable of.
The many command-line tools offer all sorts of possibilities that are well suited to the variety of tasks encompassing data science.</p>
<p>There are many definitions for data science available.
In <a href="#chapter-1-introduction">Chapter 1</a>, I introduced the OSEMN model as defined by Mason and Wiggins, because it is a very practical one that translates to very specific tasks.
The acronym OSEMN stands for obtaining, scrubbing, exploring, modeling, and interpreting data. <a href="#chapter-1-introduction">Chapter 1</a> also explained why the command line is very suitable for doing these data science tasks.</p>
<p>In <a href="#chapter-2-getting-started">Chapter 2</a>, I explained how you can get all the tools used in this book. <a href="#chapter-2-getting-started">Chapter 2</a> also provided an introduction to the essential tools and concepts of the command line.</p>
<p>The four OSEMN model chapters focused on performing those practical tasks using the command line.
I haven’t devoted a chapter to the fifth step, interpreting data, because, quite frankly, the computer, let alone the command line, is of very little use here.
I have, however, provided some pointers for further reading on this topic.</p>
<p>In the four intermezzo chapters, we looked at some broader topics of doing data science at the command line, topics which are not really specific to one particular step.
In <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>, I explained how you can turn one-liners and existing code into reusable command-line tools.
In <a href="#chapter-6-project-management-with-make">Chapter 6</a>, I described how you can manage your data workflow using a tool called <code>make</code>.
In <a href="#chapter-8-parallel-pipelines">Chapter 8</a>, I demonstrated how ordinary command-line tools and pipelines can be run in parallel using GNU Parallel.
In <a href="#chapter-10-polyglot-data-science">Chapter 10</a>, I showed that the command line doesn’t exist in a vacuum but that it can be leveraged from other programming languages and environments.
The topics discussed in these intermezzo chapters can be applied at any point in your data workflow.</p>
<p>It’s impossible to demonstrate all command-line tools that are available and relevant for doing data science.
New tools are created on a daily basis.
As you may have come to understand by now, this book is more about the idea of using the command line, rather than giving you an exhaustive list of tools.</p>
</div>
<div id="three-pieces-of-advice" class="section level2" number="11.2">
<h2 number="11.2"><span class="header-section-number">11.2</span> Three Pieces of Advice</h2>
<p>You probably spent quite some time reading these chapters and perhaps also following along with the code examples.
In the hope that it maximizes the return on this investment and increases the probability that you’ll continue to incorporate the command line into your data science workflow, I would like to offer you three pieces of advice: (1) be patient, (2) be creative, and (3) be practical. In the next three subsections I elaborate on each piece of advice.</p>
<div id="be-patient" class="section level3" number="11.2.1">
<h3 number="11.2.1"><span class="header-section-number">11.2.1</span> Be Patient</h3>
<p>The first piece of advice that I can give is to be patient.
Working with data on the command line is different from using a programming language, and therefore it requires a different mindset.</p>
<p>Moreover, the command-line tools themselves are not without their quirks and inconsistencies.
This is partly because they have been developed by many different people, over the course of multiple decades.
If you ever find yourself at a loss regarding their mind-dazzling options, don’t forget to use <code>--help</code>, <code>man</code>, <code>tldr</code>, or your favorite search engine to learn more.</p>
<p>Still, especially in the beginning, it can be a frustrating experience.
Trust me, you’ll become more proficient as you practice using the command line and its tools.
The command line has been around for many decades, and will be around for many more to come.
It’s a worthwhile investment.</p>
</div>
<div id="be-creative" class="section level3" number="11.2.2">
<h3 number="11.2.2"><span class="header-section-number">11.2.2</span> Be Creative</h3>
<p>The second, related piece of advice is to be creative.
The command line is very flexible.
By combining the command-line tools, you can accomplish more than you might think.</p>
<p>I encourage you to not immediately fall back onto your programming language.
And when you do have to use a programming language, think about whether the code can be generalized or reused in some way.
If so, consider creating your own command-line tool with that code using the steps I discussed in <a href="#chapter-4-creating-command-line-tools">Chapter 4</a>.
If you believe your tool may be beneficial for others, you could even go one step further by making it open source.
Maybe there’s a step you know how to perform at the command line, but you would rather not leave the comfort of the main programming language or environment you’re working in.
Perhaps you can use one of the approaches listed in <a href="#chapter-10-polyglot-data-science">Chapter 10</a>.</p>
</div>
<div id="be-practical" class="section level3" number="11.2.3">
<h3 number="11.2.3"><span class="header-section-number">11.2.3</span> Be Practical</h3>
<p>The third piece of advice is to be practical.
Being practical is related to being creative, but deserves a separate explanation.
In the previous subsection, I mentioned that you should not immediately fall back to a programming language.
Of course, the command line has its limits.
Throughout the book, I have emphasized that the command line should be regarded as a companion approach to doing data science.</p>
<p>I’ve discussed four steps for doing data science at the command line.
In practice, the applicability of the command line is higher for step 1 than it is for step 4.
You should use whatever approach works best for the task at hand.
And it’s perfectly fine to mix and match approaches at any point in your workflow.
As I’ve shown in <a href="#chapter-10-polyglot-data-science">Chapter 10</a>, the command line is wonderful at being integrated with other approaches, programming languages, and statistical environments.
There’s a certain trade-off with each approach, and part of becoming proficient at the command line is to learn when to use which.</p>
<p>In conclusion, when you’re patient, creative, and practical, the command line will make you a more efficient and productive data scientist.</p>
</div>
</div>
<div id="where-to-go-from-here" class="section level2" number="11.3">
<h2 number="11.3"><span class="header-section-number">11.3</span> Where To Go From Here?</h2>
<p>As this book is on the intersection of the command line and data science, many related topics have only been touched upon.
Now, it’s up to you to further explore these topics.
The following subsections provide a list of topics and suggested resources to consult.</p>
</div>
<div id="the-command-line" class="section level2" number="11.4">
<h2 number="11.4"><span class="header-section-number">11.4</span> The Command Line</h2>
<ul>
<li><em>The Linux Command Line: A Complete Introduction, 2nd Edition</em> By William E. Shotts, Jr. (No Starch Press, 2019)</li>
<li><em>Unix Power Tools, 3rd Edition</em> by Jerry Peek, Shelley Powers, Tim O’Reilly, and Mike Loukides (O’Reilly Media, 2002)</li>
<li><em>Learning the Vi and Vim Editors, 7th Edition</em> by Arnold Robbins, Elbert Hannah, and Linda Lamb (O’Reilly Media, 2008)</li>
</ul>
<div id="shell-programming" class="section level3" number="11.4.1">
<h3 number="11.4.1"><span class="header-section-number">11.4.1</span> Shell Programming</h3>
<ul>
<li><em>Classic Shell Scripting</em> by Arnold Robbins and Nelson H.F. Beebe (O’Reilly Media, 2005)</li>
<li><em>Wicked Cool Shell Scripts, 2nd Edition</em> by Dave Taylor and Brandon Perry (No Starch Press, 2017)</li>
<li><em>Bash Cookbook</em> by Carl Albing JP Vossen (O’Reilly Media, 2018)</li>
</ul>
</div>
<div id="python-r-and-sql" class="section level3" number="11.4.2">
<h3 number="11.4.2"><span class="header-section-number">11.4.2</span> Python, R, and SQL</h3>
<ul>
<li><em>Learn Python 3 the Hard Way</em> by Zed A. Shaw (Addison-Wesley Professional, 2017)</li>
<li><em>Python for Data Analysis, 2nd Edition</em> by Wes McKinney (O’Reilly Media, 2017)</li>
<li><em>Data Science from Scratch, 2nd Edition</em> by Joel Grus (O’Reilly Media, 2019)</li>
<li><em>R for Data Science</em> by Garrett Grolemund and Hadley Wickham (O’Reilly Media, 2016)</li>
<li><em>R for Everyone, 2nd edition</em> by Jared Lander (Addison-Wesley Professional, 2017)</li>
<li><em>Sams Teach Yourself SQL in 10 Minutes a Day, 5th Edition</em> by Ben Forta (Sams, 2020)</li>
</ul>
</div>
<div id="apis" class="section level3" number="11.4.3">
<h3 number="11.4.3"><span class="header-section-number">11.4.3</span> APIs</h3>
<ul>
<li><em>Mining the Social Web, 3rd Edition</em> by Matthew A. Russell and Mikhail Klassen (O’Reilly Media, 2019)</li>
<li><em>Data Source Handbook</em> by Pete Warden (O’Reilly Media, 2011)</li>
</ul>
</div>
<div id="machine-learning" class="section level3" number="11.4.4">
<h3 number="11.4.4"><span class="header-section-number">11.4.4</span> Machine Learning</h3>
<ul>
<li><em>Python Machine Learning, 3rd Edition</em> by Sebastian Raschka and Vahid Mirjalili (Packt Publishing, 2019)</li>
<li><em>Pattern Recognition and Machine Learning</em> by Christopher M. Bishop (Springer, 2006)</li>
<li><em>Information Theory, Inference, and Learning Algorithms</em> by David MacKay (Cambridge University Press, 2003)</li>
</ul>
</div>
</div>
<div id="getting-in-touch" class="section level2" number="11.5">
<h2 number="11.5"><span class="header-section-number">11.5</span> Getting in Touch</h2>
<p>This book would not have been possible without the many people who created the command line and the numerous tools.
It’s safe to say that the current ecosystem of command-line tools for data science is a community effort.
I have only been able to give you a glimpse of the many command-line tools available.
New ones are created everyday, and perhaps some day you will create one yourself.
In that case, I would love to hear from you.
I’d also appreciate it if you would drop me a line whenever you have a question, comment, or suggestion.
There are a couple of ways to get in touch:</p>
<ul>
<li>Email: <a href="mailto:jeroen@jeroenjanssens.com" class="email">jeroen@jeroenjanssens.com</a></li>
<li>Twitter: <a href="https://twitter.com/jeroenhjanssens/">@jeroenhjanssens</a></li>
<li>Book website: <a href="https://datascienceatthecommandline.com/" class="uri">https://datascienceatthecommandline.com/</a></li>
<li>Book GitHub repository: <a href="https://github.com/jeroenjanssens/data-science-at-the-command-line" class="uri">https://github.com/jeroenjanssens/data-science-at-the-command-line</a></li>
</ul>
<p>Thank you.</p>
<!--chapter:end:11.Rmd-->
<!--A[appendix]
[[appendix-tools]]
A-->
</div>
</div>
<div id="list-of-command-line-tools" class="section level1 unnumbered">
<h1 class="unnumbered">List of Command-Line Tools</h1>
<p>This is an overview of all the command-line tools discussed in this book.
This includes binary executables, interpreted scripts, and Z Shell builtins and keywords.
For each command-line tool, the following information, when available and appropriate, is provided:</p>
<ul>
<li>The actual command to type at the command line</li>
<li>A description</li>
<li>The version used in the book</li>
<li>The year that version was released</li>
<li>The primary author(s)</li>
<li>A website to find more information</li>
<li>How to obtain help</li>
<li>An example usage</li>
</ul>
<p>All command-line tools listed here are included in the Docker image.
See <a href="#chapter-2-getting-started">Chapter 2</a> for instructions on how to set it up.
Please note that citing open source software is not trivial, and that some information may be missing or incorrect.</p>
<div id="alias" class="section level2 unnumbered">
<h2 class="unnumbered">alias</h2>
<p>Define or display aliases.
<code>alias</code>
is a Z shell builtin.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> alias
alias is a shell builtin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zshbuiltins | <span style="color: #5f8700">grep</span> -A 10 alias
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">alias</span> l
l='ls --color -lhF --group-directories-first'
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">alias</span> python=python3</pre>
</div>
<div id="awk" class="section level2 unnumbered">
<h2 class="unnumbered">awk</h2>
<p>Pattern scanning and text processing language.
<code>awk</code>
(version 1.3.4)
by Mike D. Brennan and Thomas E. Dickey (2019).
More information: <a href="https://invisible-island.net/mawk" class="uri">https://invisible-island.net/mawk</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> awk
awk is /usr/bin/awk
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> awk
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">awk</span> <span style="color: #af8700">'{sum+=$1} END {print sum}'</span>
15</pre>
</div>
<div id="aws" class="section level2 unnumbered">
<h2 class="unnumbered">aws</h2>
<p>Unified tool to manage AWS services.
<code>aws</code>
(version 2.1.32)
by Amazon Web Services (2021).
More information: <a href="https://aws.amazon.com/cli" class="uri">https://aws.amazon.com/cli</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> aws
aws is /usr/local/bin/aws
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">aws</span> --help</pre>
</div>
<div id="bash" class="section level2 unnumbered">
<h2 class="unnumbered">bash</h2>
<p>GNU Bourne-Again SHell.
<code>bash</code>
(version 5.0.17)
by Brian Fox and Chet Ramey (2019).
More information: <a href="https://www.gnu.org/software/bash" class="uri">https://www.gnu.org/software/bash</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> bash
bash is /usr/bin/bash
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> bash</pre>
</div>
<div id="bat" class="section level2 unnumbered">
<h2 class="unnumbered">bat</h2>
<p>A cat clone with syntax highlighting and Git integration.
<code>bat</code>
(version 0.18.0)
by David Peter (2021).
More information: <a href="https://github.com/sharkdp/bat" class="uri">https://github.com/sharkdp/bat</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> bat
bat is an alias for bat --tabs 8 --paging never
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> bat</pre>
</div>
<div id="bc" class="section level2 unnumbered">
<h2 class="unnumbered">bc</h2>
<p>An arbitrary precision calculator language.
<code>bc</code>
(version 1.07.1)
by Philip A. Nelson (2017).
More information: <a href="https://www.gnu.org/software/bc" class="uri">https://www.gnu.org/software/bc</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> bc
bc is /usr/bin/bc
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> bc
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">bc</span> -l <span style="color: #af8700">&lt;&lt;&lt;</span> <span style="color: #af8700">'e(1)'</span>
2.71828182845904523536</pre>
</div>
<div id="body" class="section level2 unnumbered">
<h2 class="unnumbered">body</h2>
<p>Apply command to all but the first line.
<code>body</code>
(version 0.1)
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> body
body is /usr/bin/dsutils/body
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 10 | <span style="color: #5f8700">header</span> -a <span style="color: #af8700">'values'</span> | <span style="color: #5f8700">body</span> shuf
values
3
7
10
9
5
4
6
8
2
1</pre>
</div>
<div id="cat" class="section level2 unnumbered">
<h2 class="unnumbered">cat</h2>
<p>Concatenate files and print on the standard output.
<code>cat</code>
(version 8.30)
by Torbjorn Granlund and Richard M. Stallman (2018).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> cat
cat is /usr/bin/cat
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> cat
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cat</span> <span style="color: #0087ff">*</span>.log <span style="color: #af8700">&gt;</span> all.log</pre>
</div>
<div id="cd" class="section level2 unnumbered">
<h2 class="unnumbered">cd</h2>
<p>Change the shell working directory.
<code>cd</code>
is a Z shell builtin.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> cd
cd is a shell builtin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zshbuiltins | <span style="color: #5f8700">grep</span> -A 10 cd
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">~</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">pwd</span>
/home/dst
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">..</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">pwd</span>
/home
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> /data/ch01
cd: no such file or directory: /data/ch01</pre>
</div>
<div id="chmod" class="section level2 unnumbered">
<h2 class="unnumbered">chmod</h2>
<p>Change file mode bits.
<code>chmod</code>
(version 8.30)
by David MacKenzie and Jim Meyering (2018).
I use <code>chmod</code> in <a href="#chapter-4-creating-command-line-tools">Chapter 4</a> to make a tool executable.
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> chmod
chmod is /usr/bin/chmod
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> chmod
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">chmod</span> u+x script.sh</pre>
</div>
<div id="cols" class="section level2 unnumbered">
<h2 class="unnumbered">cols</h2>
<p>Apply command to subset of columns.
<code>cols</code>
(version 0.1)
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> cols
cols is /usr/bin/dsutils/cols</pre>
</div>
<div id="column" class="section level2 unnumbered">
<h2 class="unnumbered">column</h2>
<p>Columnate lists.
<code>column</code>
(version 2.36.1)
by Karel Zak (2021).
More information: <a href="https://www.kernel.org/pub/linux/utils/util-linux" class="uri">https://www.kernel.org/pub/linux/utils/util-linux</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> column
column is /usr/bin/column</pre>
</div>
<div id="cowsay" class="section level2 unnumbered">
<h2 class="unnumbered">cowsay</h2>
<p>Configurable speaking cow.
<code>cowsay</code>
(version 3.0.3)
by Tony Monroe (1999).
More information: <a href="https://github.com/tnalpgge/rank-amateur-cowsay" class="uri">https://github.com/tnalpgge/rank-amateur-cowsay</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> cowsay
cowsay is /usr/bin/cowsay
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> cowsay
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'The command line is awesome!'</span> | <span style="color: #5f8700">cowsay</span>
 ______________________________
&lt; The command line is awesome! &gt;
 ------------------------------
        \   ^__^
         \  (oo)\_______
            (__)\       )\/\
                ||----w |
                ||     ||</pre>
</div>
<div id="cp" class="section level2 unnumbered">
<h2 class="unnumbered">cp</h2>
<p>Copy files and directories.
<code>cp</code>
(version 8.30)
by Torbjorn Granlund, David MacKenzie, and Jim Meyering (2018).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> cp
cp is /usr/bin/cp
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> cp
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cp</span> -r ~/Downloads/<span style="color: #0087ff">*</span>.xlsx <span style="text-decoration: underline">/data</span></pre>
</div>
<div id="csv2vw" class="section level2 unnumbered">
<h2 class="unnumbered">csv2vw</h2>
<p>Convert CSV to Vowpal Wabbit format.
<code>csv2vw</code>
(version 0.1)
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csv2vw
csv2vw is /usr/bin/dsutils/csv2vw</pre>
</div>
<div id="csvcut" class="section level2 unnumbered">
<h2 class="unnumbered">csvcut</h2>
<p>Filter and truncate CSV files.
<code>csvcut</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvcut
csvcut is /usr/bin/csvcut
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvcut</span> --help
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvcut</span> -c bill,tip <span style="text-decoration: underline">/data/ch05/tips.csv</span> | <span style="color: #5f8700">trim</span>
bill,tip
16.99,1.01
10.34,1.66
21.01,3.5
23.68,3.31
24.59,3.61
25.29,4.71
8.77,2.0
26.88,3.12
15.04,1.96
… with 235 more lines</pre>
</div>
<div id="csvgrep" class="section level2 unnumbered">
<h2 class="unnumbered">csvgrep</h2>
<p>Search CSV files.
<code>csvgrep</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvgrep
csvgrep is /usr/bin/csvgrep
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvgrep</span> --help</pre>
</div>
<div id="csvjoin" class="section level2 unnumbered">
<h2 class="unnumbered">csvjoin</h2>
<p>Execute a SQL-like join to merge CSV files on a specified column or columns.
<code>csvjoin</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvjoin
csvjoin is /usr/bin/csvjoin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvjoin</span> --help</pre>
</div>
<div id="csvlook" class="section level2 unnumbered">
<h2 class="unnumbered">csvlook</h2>
<p>Render a CSV file in the console as a Markdown-compatible, fixed-width table.
<code>csvlook</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvlook
csvlook is a shell function
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> --help
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvlook</span> <span style="text-decoration: underline">/data/ch05/tips.csv</span>
│  bill │   tip │ sex    │ smoker │ day  │ time   │ size │
├───────┼───────┼────────┼────────┼──────┼────────┼──────┤
│ 16.99 │  1.01 │ Female │  False │ Sun  │ Dinner │    2 │
│ 10.34 │  1.66 │ Male   │  False │ Sun  │ Dinner │    3 │
│ 21.01 │  3.50 │ Male   │  False │ Sun  │ Dinner │    3 │
│ 23.68 │  3.31 │ Male   │  False │ Sun  │ Dinner │    2 │
│ 24.59 │  3.61 │ Female │  False │ Sun  │ Dinner │    4 │
│ 25.29 │  4.71 │ Male   │  False │ Sun  │ Dinner │    4 │
│  8.77 │  2.00 │ Male   │  False │ Sun  │ Dinner │    2 │
│ 26.88 │  3.12 │ Male   │  False │ Sun  │ Dinner │    4 │
… with 236 more lines</pre>
</div>
<div id="csvquote" class="section level2 unnumbered">
<h2 class="unnumbered">csvquote</h2>
<p>Enable common unix utlities to work correctly with CSV data.
<code>csvquote</code>
(version 0.1)
by Dan Brown (2018).
More information: <a href="https://github.com/dbro/csvquote" class="uri">https://github.com/dbro/csvquote</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvquote
csvquote is /usr/local/bin/csvquote</pre>
</div>
<div id="csvsort" class="section level2 unnumbered">
<h2 class="unnumbered">csvsort</h2>
<p>Sort CSV files.
<code>csvsort</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvsort
csvsort is /usr/bin/csvsort
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvsort</span> --help</pre>
</div>
<div id="csvsql" class="section level2 unnumbered">
<h2 class="unnumbered">csvsql</h2>
<p>Execute SQL statements on CSV files.
<code>csvsql</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvsql
csvsql is /usr/bin/csvsql
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvsql</span> --help</pre>
</div>
<div id="csvstack" class="section level2 unnumbered">
<h2 class="unnumbered">csvstack</h2>
<p>Stack up the rows from multiple CSV files.
<code>csvstack</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvstack
csvstack is /usr/bin/csvstack
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstack</span> --help</pre>
</div>
<div id="csvstat" class="section level2 unnumbered">
<h2 class="unnumbered">csvstat</h2>
<p>Print descriptive statistics for each column in a CSV file.
<code>csvstat</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> csvstat
csvstat is /usr/bin/csvstat
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">csvstat</span> --help</pre>
</div>
<div id="curl" class="section level2 unnumbered">
<h2 class="unnumbered">curl</h2>
<p>Transfer a URL.
<code>curl</code>
(version 7.68.0)
by Daniel Stenberg (2016).
More information: <a href="https://curl.haxx.se" class="uri">https://curl.haxx.se</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> curl
curl is /usr/bin/curl
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> curl</pre>
</div>
<div id="cut" class="section level2 unnumbered">
<h2 class="unnumbered">cut</h2>
<p>Remove sections from each line of files.
<code>cut</code>
(version 8.30)
by David M. Ihnat, David MacKenzie, and Jim Meyering (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> cut
cut is /usr/bin/cut
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> cut</pre>
</div>
<div id="display" class="section level2 unnumbered">
<h2 class="unnumbered">display</h2>
<p>Displays an image or image sequence on any X server.
<code>display</code>
(version 6.9.10-23)
by ImageMagick Studio LLC (2019).
More information: <a href="https://imagemagick.org" class="uri">https://imagemagick.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> display
display is a shell function</pre>
</div>
<div id="dseq" class="section level2 unnumbered">
<h2 class="unnumbered">dseq</h2>
<p>Generate sequence of dates.
<code>dseq</code>
(version 0.1)
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> dseq
dseq is /usr/bin/dsutils/dseq
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">dseq</span> 3
2021-12-15
2021-12-16
2021-12-17</pre>
</div>
<div id="echo" class="section level2 unnumbered">
<h2 class="unnumbered">echo</h2>
<p>Display a line of text.
<code>echo</code>
(version 8.30)
by Brian Fox and Chet Ramey (2019).
Useful for using literal text as standard input to the next tool.
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> echo
echo is a shell builtin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> echo
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> Hippopotomonstrosesquippedaliophobia | <span style="color: #5f8700">cowsay</span>
 ______________________________________
&lt; Hippopotomonstrosesquippedaliophobia &gt;
 --------------------------------------
        \   ^__^
         \  (oo)\_______
            (__)\       )\/\
                ||----w |
                ||     ||
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> -n Hippopotomonstrosesquippedaliophobia | <span style="color: #5f8700">wc</span> -c
36</pre>
</div>
<div id="env" class="section level2 unnumbered">
<h2 class="unnumbered">env</h2>
<p>Run a program in a modified environment.
<code>env</code>
(version 8.30)
by Richard Mlynarik, David MacKenzie, and Assaf Gordon (2018).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> env
env is /usr/bin/env
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> env</pre>
</div>
<div id="export" class="section level2 unnumbered">
<h2 class="unnumbered">export</h2>
<p>Set export attribute for shell variables. Useful for making shell variables available to other command-line tools..
<code>export</code>
is a Z shell builtin.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> export
export is a reserved word
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zshbuiltins | <span style="color: #5f8700">grep</span> -A 10 export
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">export</span> PATH=<span style="color: #af8700">"</span><span style="color: #00afaf">$PATH</span><span style="color: #af8700">:</span><span style="color: #00afaf">$HOME</span><span style="color: #af8700">/bin"</span></pre>
</div>
<div id="fc" class="section level2 unnumbered">
<h2 class="unnumbered">fc</h2>
<p>Control the interactive history mechanism.
<code>fc</code>
is a Z shell builtin.
I use <code>fc</code> in <a href="#chapter-4-creating-command-line-tools">Chapter 4</a> to edit the command in <code>nano</code>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> fc
fc is a shell builtin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zshbuiltins | <span style="color: #5f8700">grep</span> -A 10 <span style="color: #af8700">'^ *fc '</span></pre>
</div>
<div id="find" class="section level2 unnumbered">
<h2 class="unnumbered">find</h2>
<p>Search for files in a directory hierarchy.
<code>find</code>
(version 4.7.0)
by Eric B. Decker, James Youngman, and Kevin Dalley (2019).
More information: <a href="https://www.gnu.org/software/findutils" class="uri">https://www.gnu.org/software/findutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> find
find is /usr/bin/find
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> find
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">find</span> <span style="text-decoration: underline">/data</span> -type f -name <span style="color: #af8700">'*.csv'</span> -size -3
/data/ch03/tmnt-basic.csv
/data/ch03/tmnt-missing-newline.csv
/data/ch03/tmnt-with-header.csv
/data/ch05/irismeta.csv
/data/ch05/names-comma.csv
/data/ch05/names.csv
/data/ch07/datatypes.csv</pre>
</div>
<div id="fold" class="section level2 unnumbered">
<h2 class="unnumbered">fold</h2>
<p>Wrap each input line to fit in specified width.
<code>fold</code>
(version 8.30)
by David MacKenzie (2020).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> fold
fold is /usr/bin/fold
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> fold</pre>
</div>
<div id="for" class="section level2 unnumbered">
<h2 class="unnumbered">for</h2>
<p>Execute commands for each member in a list.
<code>for</code>
is a Z shell builtin.
In <a href="#chapter-8-parallel-pipelines">Chapter 8</a>, I discuss the advantages of using <code>parallel</code> instead of <code>for</code>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> for
for is a reserved word
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zshmisc | <span style="color: #5f8700">grep</span> -EA 10 <span style="color: #af8700">'^ *for '</span>
 
<span style="font-weight: bold">$</span> <span style="color: #af8700">for</span> i in {A..C} <span style="color: #af8700">"It's easy as"</span> {1..3}; <span style="color: #af8700">do</span> <span style="color: #5f8700">echo</span> $i; <span style="color: #af8700">done</span>
A
B
C
It's easy as
1
2
3</pre>
</div>
<div id="fx" class="section level2 unnumbered">
<h2 class="unnumbered">fx</h2>
<p>Interactive JSON viewer.
<code>fx</code>
(version 20.0.2)
by Anton Medvedev (2020).
More information: <a href="https://github.com/antonmedv/fx" class="uri">https://github.com/antonmedv/fx</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> fx
fx is /usr/local/bin/fx
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">fx</span> --help
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'[1,2,3]'</span> | <span style="color: #5f8700">fx</span> <span style="color: #af8700">'this.map(x =&gt; x * 2)'</span>
[
  <span style="font-weight: bold"></span><span style="font-weight: bold; color: #00afaf">2</span>,
  <span style="font-weight: bold"></span><span style="font-weight: bold; color: #00afaf">4</span>,
  <span style="font-weight: bold"></span><span style="font-weight: bold; color: #00afaf">6</span>
]</pre>
</div>
<div id="git" class="section level2 unnumbered">
<h2 class="unnumbered">git</h2>
<p>The stupid content tracker.
<code>git</code>
(version 2.25.1)
by Linus Torvalds and Junio C. Hamano (2021).
More information: <a href="https://git-scm.com" class="uri">https://git-scm.com</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> git
git is /usr/bin/git
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> git</pre>
</div>
<div id="grep" class="section level2 unnumbered">
<h2 class="unnumbered">grep</h2>
<p>Print lines that match patterns.
<code>grep</code>
(version 3.4)
by Jim Meyering (2019).
More information: <a href="https://www.gnu.org/software/grep" class="uri">https://www.gnu.org/software/grep</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> grep
grep is /usr/bin/grep
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> grep
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 100 | <span style="color: #5f8700">grep</span> 3 | <span style="color: #5f8700">wc</span> -l
19</pre>
</div>
<div id="gron" class="section level2 unnumbered">
<h2 class="unnumbered">gron</h2>
<p>Make JSON greppable.
<code>gron</code>
(version 0.6.1)
by Tom Hudson (2021).
More information: <a href="https://github.com/TomNomNom/gron" class="uri">https://github.com/TomNomNom/gron</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> gron
gron is /usr/bin/gron
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> gron</pre>
</div>
<div id="head" class="section level2 unnumbered">
<h2 class="unnumbered">head</h2>
<p>Output the first part of files.
<code>head</code>
(version 8.30)
by David MacKenzie and Jim Meyering (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> head
head is /usr/bin/head
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> head
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 100 | <span style="color: #5f8700">head</span> -n 5
1
2
3
4
5</pre>
</div>
<div id="header" class="section level2 unnumbered">
<h2 class="unnumbered">header</h2>
<p>Add, replace, and delete header lines.
<code>header</code>
(version 0.1)
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> header
header is /usr/bin/dsutils/header</pre>
</div>
<div id="history" class="section level2 unnumbered">
<h2 class="unnumbered">history</h2>
<p>GNU History Library.
<code>history</code>
(version 8.1)
by Brian Fox and Chet Ramey (2020).
More information: <a href="https://www.gnu.org/software/bash" class="uri">https://www.gnu.org/software/bash</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> history
history is a shell builtin</pre>
</div>
<div id="hostname" class="section level2 unnumbered">
<h2 class="unnumbered">hostname</h2>
<p>Show or set the system’s host name.
<code>hostname</code>
(version 3.23)
by Peter Tobias, Bernd Eckenfels, and Michael Meskes (2021).
More information: <a href="https://sourceforge.net/projects/net-tools/" class="uri">https://sourceforge.net/projects/net-tools/</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> hostname
hostname is /usr/bin/hostname
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> hostname
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">hostname</span>
2c3edb64827b
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">hostname</span> -i
172.17.0.2</pre>
</div>
<div id="in2csv" class="section level2 unnumbered">
<h2 class="unnumbered">in2csv</h2>
<p>Convert common, but less awesome, tabular data formats to CSV.
<code>in2csv</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> in2csv
in2csv is /usr/bin/in2csv
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">in2csv</span> --help</pre>
</div>
<div id="jq" class="section level2 unnumbered">
<h2 class="unnumbered">jq</h2>
<p>Command-line JSON processor.
<code>jq</code>
(version 1.6)
by Stephen Dolan (2021).
More information: <a href="https://stedolan.github.com/jq" class="uri">https://stedolan.github.com/jq</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> jq
jq is /usr/bin/jq
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> jq</pre>
</div>
<div id="json2csv" class="section level2 unnumbered">
<h2 class="unnumbered">json2csv</h2>
<p>Convert JSON to CSV.
<code>json2csv</code>
(version 1.2.1)
by Jehiah Czebotar (2019).
More information: <a href="https://github.com/jehiah/json2csv" class="uri">https://github.com/jehiah/json2csv</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> json2csv
json2csv is /usr/bin/json2csv
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">json2csv</span> --help</pre>
</div>
<div id="l" class="section level2 unnumbered">
<h2 class="unnumbered">l</h2>
<p>List directory contents in long format with directories grouped before files, human readable file sizes, and access permissions.
<code>l</code>
by Unknown (1999).</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> l
l is an alias for ls --color -lhF --group-directories-first
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data/ch03</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">ls</span>
logs.tar.gz    tmnt-basic.csv            tmnt-with-header.csv
r-datasets.db  tmnt-missing-newline.csv  top2000.xlsx
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">l</span>
total 924K
-rw-r--r-- 1 dst dst 627K Dec 14 12:03 <span style="font-weight: bold"></span><span style="font-weight: bold; color: #d70000">logs.tar.gz</span>
-rw-r--r-- 1 dst dst 189K Dec 14 12:03 r-datasets.db
-rw-r--r-- 1 dst dst  149 Dec 14 12:03 tmnt-basic.csv
-rw-r--r-- 1 dst dst  148 Dec 14 12:03 tmnt-missing-newline.csv
-rw-r--r-- 1 dst dst  181 Dec 14 12:03 tmnt-with-header.csv
-rw-r--r-- 1 dst dst  91K Dec 14 12:03 top2000.xlsx</pre>
</div>
<div id="less" class="section level2 unnumbered">
<h2 class="unnumbered">less</h2>
<p>opposite of more.
<code>less</code>
(version 551)
by Mark Nudelman (2019).
More information: <a href="https://www.greenwoodsoftware.com/less" class="uri">https://www.greenwoodsoftware.com/less</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> less
less is an alias for less -R
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> less
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">less</span> README</pre>
</div>
<div id="ls" class="section level2 unnumbered">
<h2 class="unnumbered">ls</h2>
<p>List directory contents.
<code>ls</code>
(version 8.30)
by Richard M. Stallman and David MacKenzie (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> ls
ls is /usr/bin/ls
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> ls
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">ls</span> <span style="text-decoration: underline">/data</span>
ch02  ch03  ch04  ch05  ch06  ch07  ch08  ch09  ch10  csvconf</pre>
</div>
<div id="make" class="section level2 unnumbered">
<h2 class="unnumbered">make</h2>
<p>A Program for Maintaining Computer Programs.
<code>make</code>
(version 4.3)
by Stuart I. Feldman (2020).
More information: <a href="https://www.gnu.org/software/make" class="uri">https://www.gnu.org/software/make</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> make
make is /usr/bin/make
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> make
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">make</span> sandwich</pre>
</div>
<div id="man" class="section level2 unnumbered">
<h2 class="unnumbered">man</h2>
<p>An interface to the system reference manuals.
<code>man</code>
(version 2.9.1)
by John W. Eaton and Colin Watson (2020).
More information: <a href="https://nongnu.org/man-db" class="uri">https://nongnu.org/man-db</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> man
man is /usr/bin/man
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> man
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> excel
No manual entry for excel</pre>
</div>
<div id="mkdir" class="section level2 unnumbered">
<h2 class="unnumbered">mkdir</h2>
<p>Make directories.
<code>mkdir</code>
(version 8.30)
by David MacKenzie (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> mkdir
mkdir is /usr/bin/mkdir
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> mkdir
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">mkdir</span> -p /data/ch{01..10}</pre>
</div>
<div id="mv" class="section level2 unnumbered">
<h2 class="unnumbered">mv</h2>
<p>Move (rename) files.
<code>mv</code>
(version 8.30)
by Mike Parker, David MacKenzie, and Jim Meyering (2020).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> mv
mv is /usr/bin/mv
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> mv
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">mv</span> results{,.bak}</pre>
</div>
<div id="nano" class="section level2 unnumbered">
<h2 class="unnumbered">nano</h2>
<p>Nano’s ANOther editor, inspired by Pico.
<code>nano</code>
(version 5.4)
by Benno Schulenberg, David Lawrence Ramsey, Jordi Mallach, Chris Allegretta, Robert Siemborski, and Adam Rogoyski (2020).
More information: <a href="https://nano-editor.org" class="uri">https://nano-editor.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> nano
nano is /usr/bin/nano</pre>
</div>
<div id="nl" class="section level2 unnumbered">
<h2 class="unnumbered">nl</h2>
<p>Number lines of files.
<code>nl</code>
(version 8.30)
by Scott Bartram and David MacKenzie (2020).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> nl
nl is /usr/bin/nl
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> nl
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">nl</span> <span style="text-decoration: underline">/data/ch05/alice.txt</span> | <span style="color: #5f8700">head</span>
     1  ﻿Project Gutenberg's Alice's Adventures in Wonderland, by Lewis Carroll
     2
     3  This eBook is for the use of anyone anywhere at no cost and with
     4  almost no restrictions whatsoever.  You may copy it, give it away or
     5  re-use it under the terms of the Project Gutenberg License included
     6  with this eBook or online at www.gutenberg.org
     7
     8
     9  Title: Alice's Adventures in Wonderland
    10</pre>
</div>
<div id="parallel" class="section level2 unnumbered">
<h2 class="unnumbered">parallel</h2>
<p>Build and execute shell command lines from standard input in parallel.
<code>parallel</code>
(version 20161222)
by Ole Tange (2016).
More information: <a href="https://www.gnu.org/software/parallel" class="uri">https://www.gnu.org/software/parallel</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> parallel
parallel is /usr/bin/parallel
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> parallel
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 3 | <span style="color: #5f8700">parallel</span> <span style="color: #af8700">"echo Processing file {}.csv"</span>
Processing file 1.csv
Processing file 2.csv
Processing file 3.csv</pre>
</div>
<div id="paste" class="section level2 unnumbered">
<h2 class="unnumbered">paste</h2>
<p>Merge lines of files.
<code>paste</code>
(version 8.30)
by David M. Ihnat and David MacKenzie (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> paste
paste is /usr/bin/paste
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> paste
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">paste</span> -d, <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">seq</span> 5<span style="color: #af005f">)</span> <span style="color: #af005f">&lt;(</span><span style="color: #5f8700">dseq</span> 5<span style="color: #af005f">)</span>
1,2021-12-15
2,2021-12-16
3,2021-12-17
4,2021-12-18
5,2021-12-19
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 5 | <span style="color: #5f8700">paste</span> -sd+
1+2+3+4+5</pre>
</div>
<div id="pbc" class="section level2 unnumbered">
<h2 class="unnumbered">pbc</h2>
<p>Parallel bc.
<code>pbc</code>
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> pbc
pbc is /usr/bin/dsutils/pbc
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 3 | <span style="color: #5f8700">pbc</span> <span style="color: #af8700">'{1}^2'</span>
1
4
9</pre>
</div>
<div id="pip" class="section level2 unnumbered">
<h2 class="unnumbered">pip</h2>
<p>A tool for installing and managing Python packages.
<code>pip</code>
(version 20.0.2)
by PyPA (2020).
More information: <a href="https://pip.pypa.io" class="uri">https://pip.pypa.io</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> pip
pip is /usr/bin/pip
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> pip
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">pip</span> install pandas
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">pip</span> freeze | <span style="color: #5f8700">grep</span> sci
scikit-learn==0.24.2
scipy==1.7.0</pre>
</div>
<div id="pup" class="section level2 unnumbered">
<h2 class="unnumbered">pup</h2>
<p>Parsing HTML at the command line.
<code>pup</code>
(version 0.4.0)
by Eric Chiang (2016).
More information: <a href="https://github.com/EricChiang/pup" class="uri">https://github.com/EricChiang/pup</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> pup
pup is /usr/bin/pup
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">pup</span> --help</pre>
</div>
<div id="pwd" class="section level2 unnumbered">
<h2 class="unnumbered">pwd</h2>
<p>Print name of working directory.
<code>pwd</code>
(version 8.30)
by Jim Meyering (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> pwd
pwd is a shell builtin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> pwd
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">~</span>
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">pwd</span>
/home/dst</pre>
</div>
<div id="python-1" class="section level2 unnumbered">
<h2 class="unnumbered">python</h2>
<p>An interpreted, interactive, object-oriented programming language.
<code>python</code>
(version 3.8.5)
by The Python Software Foundation (2021).
More information: <a href="https://www.python.org" class="uri">https://www.python.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> python
python is an alias for python3
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> python</pre>
</div>
<div id="r-1" class="section level2 unnumbered">
<h2 class="unnumbered">R</h2>
<p>A Language and Environment for Statistical Computing.
<code>R</code>
(version 4.0.4)
by The R Foundation for Statistical Computing (2021).
More information: <a href="https://www.r-project.org" class="uri">https://www.r-project.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> R
R is /usr/bin/R
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> R</pre>
</div>
<div id="rev" class="section level2 unnumbered">
<h2 class="unnumbered">rev</h2>
<p>Reverse lines characterwise.
<code>rev</code>
(version 2.36.1)
by Karel Zak (2021).
More information: <a href="https://www.kernel.org/pub/linux/utils/util-linux" class="uri">https://www.kernel.org/pub/linux/utils/util-linux</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> rev
rev is /usr/bin/rev
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'Satire: Veritas'</span> | <span style="color: #5f8700">rev</span>
satireV :eritaS
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'Ça va?'</span> | <span style="color: #5f8700">rev</span> | <span style="color: #5f8700">cut</span> -c 2- | <span style="color: #5f8700">rev</span>
Ça va</pre>
</div>
<div id="rm" class="section level2 unnumbered">
<h2 class="unnumbered">rm</h2>
<p>Remove files or directories.
<code>rm</code>
(version 8.30)
by Paul Rubin, David MacKenzie, Richard M. Stallman, and Jim Meyering (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> rm
rm is /usr/bin/rm
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> rm
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">rm</span> <span style="color: #0087ff">*</span>.old</pre>
</div>
<div id="rush" class="section level2 unnumbered">
<h2 class="unnumbered">rush</h2>
<p>R One-Liners from the Shell.
<code>rush</code>
(version 0.1)
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/rush" class="uri">https://github.com/jeroenjanssens/rush</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> rush
rush is /usr/local/lib/R/site-library/rush/exec/rush
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> --help
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run <span style="color: #af8700">'6*7'</span>
42
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">rush</span> run --tidyverse <span style="color: #af8700">'filter(starwars, species == "Human") %&gt;% select(name)'</span>
# A tibble: 35 x 1
   name
   &lt;chr&gt;
 1 Luke Skywalker
 2 Darth Vader
 3 Leia Organa
 4 Owen Lars
 5 Beru Whitesun lars
 6 Biggs Darklighter
 7 Obi-Wan Kenobi
 8 Anakin Skywalker
 9 Wilhuff Tarkin
10 Han Solo
# … with 25 more rows</pre>
</div>
<div id="sample" class="section level2 unnumbered">
<h2 class="unnumbered">sample</h2>
<p>Filter lines from standard input according to some probability, with a given delay, and for a certain duration.
<code>sample</code>
(version 0.2.4)
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/sample" class="uri">https://github.com/jeroenjanssens/sample</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> sample
sample is /usr/local/bin/sample
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">sample</span> --help
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 1000 | <span style="color: #5f8700">sample</span> -r 0.01 | <span style="color: #5f8700">trim</span> 5
160
300
337
340
434
… with 6 more lines</pre>
</div>
<div id="scp" class="section level2 unnumbered">
<h2 class="unnumbered">scp</h2>
<p>OpenSSH secure file copy.
<code>scp</code>
(version 1:8.2p1-4ubuntu0.2)
by Timo Rinne and Tatu Ylonen (2019).
More information: <a href="https://www.openssh.com" class="uri">https://www.openssh.com</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> scp
scp is /usr/bin/scp
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> scp</pre>
</div>
<div id="sed" class="section level2 unnumbered">
<h2 class="unnumbered">sed</h2>
<p>Stream editor for filtering and transforming text.
<code>sed</code>
(version 4.7)
by Jay Fenlason, Tom Lord, Ken Pizzini, and Paolo Bonzini (2018).
More information: <a href="https://www.gnu.org/software/sed" class="uri">https://www.gnu.org/software/sed</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> sed
sed is /usr/bin/sed
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> sed</pre>
</div>
<div id="seq" class="section level2 unnumbered">
<h2 class="unnumbered">seq</h2>
<p>Print a sequence of numbers.
<code>seq</code>
(version 8.30)
by Ulrich Drepper (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> seq
seq is /usr/bin/seq
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> seq
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 3
1
2
3
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">seq</span> 10 5 20
10
15
20</pre>
</div>
<div id="servewd" class="section level2 unnumbered">
<h2 class="unnumbered">servewd</h2>
<p>Serve the current working directory using a simple HTTP server.
<code>servewd</code>
(version 0.1)
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> servewd
servewd is /usr/bin/dsutils/servewd
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">servewd</span> --help
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">cd</span> <span style="text-decoration: underline">/data</span> &amp;&amp; <span style="color: #5f8700">servewd</span> 8000</pre>
</div>
<div id="shuf" class="section level2 unnumbered">
<h2 class="unnumbered">shuf</h2>
<p>Generate random permutations.
<code>shuf</code>
(version 8.30)
by Paul Eggert (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> shuf
shuf is /usr/bin/shuf
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> shuf
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> {a..z} | <span style="color: #5f8700">tr</span> <span style="color: #af8700">' '</span> <span style="color: #af8700">'\n'</span> | <span style="color: #5f8700">shuf</span> | <span style="color: #5f8700">trim</span> 5
s
a
k
x
y
… with 21 more lines
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">shuf</span> -i 1-100 | <span style="color: #5f8700">trim</span> 5
61
60
5
92
10
… with 95 more lines</pre>
</div>
<div id="skll" class="section level2 unnumbered">
<h2 class="unnumbered">skll</h2>
<p>SciKit-Learn Laboratory.
<code>skll</code>
(version 2.5.0)
by Educational Testing Service (2021).
The actual tool is <code>run_experiment</code>. I use the alias <code>skll</code> because I find that easier to remember.
More information: <a href="https://skll.readthedocs.org" class="uri">https://skll.readthedocs.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> skll
skll is an alias for run_experiment
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">skll</span> --help</pre>
</div>
<div id="sort" class="section level2 unnumbered">
<h2 class="unnumbered">sort</h2>
<p>Sort lines of text files.
<code>sort</code>
(version 8.30)
by Mike Haertel and Paul Eggert (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> sort
sort is /usr/bin/sort
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> sort
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> <span style="color: #af8700">'3\n7\n1\n3'</span> | <span style="color: #5f8700">sort</span>
1
3
3
7</pre>
</div>
<div id="split" class="section level2 unnumbered">
<h2 class="unnumbered">split</h2>
<p>Split a file into pieces.
<code>split</code>
(version 8.30)
by Torbjorn Granlund and Richard M. Stallman (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> split
split is /usr/bin/split
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> split</pre>
</div>
<div id="sponge" class="section level2 unnumbered">
<h2 class="unnumbered">sponge</h2>
<p>Soak up standard input and write to a file.
<code>sponge</code>
(version 0.65)
by Colin Watson and Tollef Fog Heen (2021).
Useful if you want to read from and write to the same file in a single pipeline.
More information: <a href="https://joeyh.name/code/moreutils" class="uri">https://joeyh.name/code/moreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> sponge
sponge is /usr/bin/sponge</pre>
</div>
<div id="sql2csv" class="section level2 unnumbered">
<h2 class="unnumbered">sql2csv</h2>
<p>Execute an SQL query on a database and output the result to a CSV file.
<code>sql2csv</code>
(version 1.0.5)
by Christopher Groskopf (2020).
More information: <a href="https://csvkit.rtfd.org" class="uri">https://csvkit.rtfd.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> sql2csv
sql2csv is /usr/bin/sql2csv
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">sql2csv</span> --help</pre>
</div>
<div id="ssh" class="section level2 unnumbered">
<h2 class="unnumbered">ssh</h2>
<p>OpenSSH remote login client.
<code>ssh</code>
(version 1:8.2p1-4ubuntu0.2)
by Tatu Ylonen, Aaron Campbell, Bob Beck, Markus Friedl, Niels Provos, Theo Raadt, Dug Song, and Markus Friedl (2020).
More information: <a href="https://www.openssh.com" class="uri">https://www.openssh.com</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> ssh
ssh is /usr/bin/ssh
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> ssh</pre>
</div>
<div id="sudo" class="section level2 unnumbered">
<h2 class="unnumbered">sudo</h2>
<p>Execute a command as another user.
<code>sudo</code>
(version 1.8.31)
by Todd C. Miller (2019).
More information: <a href="https://www.sudo.ws" class="uri">https://www.sudo.ws</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> sudo
sudo is /usr/bin/sudo
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> sudo</pre>
</div>
<div id="tail" class="section level2 unnumbered">
<h2 class="unnumbered">tail</h2>
<p>Output the last part of files.
<code>tail</code>
(version 8.30)
by Paul Rubin, David MacKenzie, Ian Lance Taylor, and Jim Meyering (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> tail
tail is /usr/bin/tail
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> tail</pre>
</div>
<div id="tapkee" class="section level2 unnumbered">
<h2 class="unnumbered">tapkee</h2>
<p>An efficient dimension reduction library.
<code>tapkee</code>
(version 1.2)
by Sergey Lisitsyn, Christian Widmer, and Fernando J. Iglesias Garcia (2013).
More information: <a href="http://tapkee.lisitsyn.me" class="uri">http://tapkee.lisitsyn.me</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> tapkee
tapkee is /usr/bin/tapkee
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">tapkee</span> --help</pre>
</div>
<div id="tar" class="section level2 unnumbered">
<h2 class="unnumbered">tar</h2>
<p>An archiving utility.
<code>tar</code>
(version 1.30)
by John Gilmore and Jay Fenlason (2014).
More information: <a href="https://www.gnu.org/software/tar" class="uri">https://www.gnu.org/software/tar</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> tar
tar is /usr/bin/tar
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> tar</pre>
</div>
<div id="tee" class="section level2 unnumbered">
<h2 class="unnumbered">tee</h2>
<p>Read from standard input and write to standard output and files.
<code>tee</code>
(version 8.30)
by Mike Parker, Richard M. Stallman, and David MacKenzie (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> tee
tee is /usr/bin/tee
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> tee</pre>
</div>
<div id="telnet" class="section level2 unnumbered">
<h2 class="unnumbered">telnet</h2>
<p>User interface to the TELNET protocol.
<code>telnet</code>
(version 0.17)
by Mats Erik Andersson, Andreas Henriksson, and Christoph Biedl (1999).
More information: <a href="http://www.hcs.harvard.edu/~dholland/computers/netkit.html" class="uri">http://www.hcs.harvard.edu/~dholland/computers/netkit.html</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> telnet
telnet is /usr/bin/telnet</pre>
</div>
<div id="tldr" class="section level2 unnumbered">
<h2 class="unnumbered">tldr</h2>
<p>Collaborative cheatsheets for console commands.
<code>tldr</code>
(version 3.3.7)
by Owen Voke (2021).
More information: <a href="https://tldr.sh" class="uri">https://tldr.sh</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> tldr
tldr is /usr/local/bin/tldr
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">tldr</span> --help
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">tldr</span> tar | <span style="color: #5f8700">trim</span>
✔ Page not found. Updating cache...
✔ Creating index...
 
  tar
 
  Archiving utility.
  Often combined with a compression method, such as gzip or bzip2.
  More information: https://www.gnu.org/software/tar.
 
  - [c]reate an archive and write it to a [f]ile:
    tar cf target.tar file1 file2 file3
 
… with 22 more lines</pre>
</div>
<div id="tr" class="section level2 unnumbered">
<h2 class="unnumbered">tr</h2>
<p>Translate or delete characters.
<code>tr</code>
(version 8.30)
by Jim Meyering (2018).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> tr
tr is /usr/bin/tr
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> tr</pre>
</div>
<div id="tree" class="section level2 unnumbered">
<h2 class="unnumbered">tree</h2>
<p>List contents of directories in a tree-like format.
<code>tree</code>
(version 1.8.0)
by Steve Baker (2018).
More information: <a href="https://launchpad.net/ubuntu/+source/tree" class="uri">https://launchpad.net/ubuntu/+source/tree</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> tree
tree is /usr/bin/tree
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> tree
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">tree</span> <span style="text-decoration: underline">/</span> | <span style="color: #5f8700">trim</span>
/
├── bin -&gt; usr/bin
├── boot
├── data
│   ├── ch02
│   │   ├── fac.py
│   │   └── movies.txt
│   ├── ch03
│   │   ├── logs.tar.gz
│   │   ├── r-datasets.db
… with 121572 more lines</pre>
</div>
<div id="trim" class="section level2 unnumbered">
<h2 class="unnumbered">trim</h2>
<p>Trim output to a given height and width.
<code>trim</code>
by Jeroen Janssens (2021).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> trim
trim is /usr/bin/dsutils/trim
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> {a..z}-{0..9} | <span style="color: #5f8700">fold</span> | <span style="color: #5f8700">trim</span> 5 60
a-0 a-1 a-2 a-3 a-4 a-5 a-6 a-7 a-8 a-9 b-0 b-1 b-2 b-3 b-4…
c-0 c-1 c-2 c-3 c-4 c-5 c-6 c-7 c-8 c-9 d-0 d-1 d-2 d-3 d-4…
e-0 e-1 e-2 e-3 e-4 e-5 e-6 e-7 e-8 e-9 f-0 f-1 f-2 f-3 f-4…
g-0 g-1 g-2 g-3 g-4 g-5 g-6 g-7 g-8 g-9 h-0 h-1 h-2 h-3 h-4…
i-0 i-1 i-2 i-3 i-4 i-5 i-6 i-7 i-8 i-9 j-0 j-1 j-2 j-3 j-4…
… with 8 more lines</pre>
</div>
<div id="ts" class="section level2 unnumbered">
<h2 class="unnumbered">ts</h2>
<p>Timestamp input.
<code>ts</code>
(version 0.65)
by Joey Hess (2021).
More information: <a href="https://joeyh.name/code/moreutils" class="uri">https://joeyh.name/code/moreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> ts
ts is /usr/bin/ts
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">echo</span> seq 5 | <span style="color: #5f8700">sample</span> -d 500 | <span style="color: #5f8700">ts</span>
Dec 14 12:07:20 seq 5</pre>
</div>
<div id="type" class="section level2 unnumbered">
<h2 class="unnumbered">type</h2>
<p>Show the type and location of a command-line tool.
<code>type</code>
is a Z shell builtin.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> type
type is a shell builtin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zshbuiltins | <span style="color: #5f8700">grep</span> -A 10 <span style="color: #af8700">'^ *type '</span></pre>
</div>
<div id="uniq" class="section level2 unnumbered">
<h2 class="unnumbered">uniq</h2>
<p>Report or omit repeated lines.
<code>uniq</code>
(version 8.30)
by Richard M. Stallman and David MacKenzie (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> uniq
uniq is /usr/bin/uniq
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> uniq</pre>
</div>
<div id="unpack" class="section level2 unnumbered">
<h2 class="unnumbered">unpack</h2>
<p>Extract common file formats.
<code>unpack</code>
(version 0.1)
by Patrick Brisbin (2013).
More information: <a href="https://github.com/jeroenjanssens/dsutils" class="uri">https://github.com/jeroenjanssens/dsutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> unpack
unpack is /usr/bin/dsutils/unpack</pre>
</div>
<div id="unrar" class="section level2 unnumbered">
<h2 class="unnumbered">unrar</h2>
<p>Extract files from rar archives.
<code>unrar</code>
(version 0.0.1)
by Ben Asselstine, Christian Scheurer, and Johannes Winkelmann (2014).
More information: <a href="http://home.gna.org/unrar" class="uri">http://home.gna.org/unrar</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> unrar
unrar is /usr/bin/unrar
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> unrar</pre>
</div>
<div id="unzip" class="section level2 unnumbered">
<h2 class="unnumbered">unzip</h2>
<p>List, test and extract compressed files in a ZIP archive.
<code>unzip</code>
(version 6.0)
by Samuel H. Smith, Ed Gordon, Christian Spieler, Onno Linden, Mike White, Kai Uwe Rommel, Steven M. Schweda, Paul Kienitz, Chris Herborth, Jonathan Hudson, Sergio Monesi, Harald Denker, John Bush, Hunter Goatley, Steve Salisbury, Steve Miller, and Dave Smith (2009).
More information: <a href="http://www.info-zip.org/pub/infozip" class="uri">http://www.info-zip.org/pub/infozip</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> unzip
unzip is /usr/bin/unzip
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> unzip</pre>
</div>
<div id="vw" class="section level2 unnumbered">
<h2 class="unnumbered">vw</h2>
<p>Fast machine learning library for online learning.
<code>vw</code>
(version 8.10.1)
by John Langford (2021).
More information: <a href="https://vowpalwabbit.org" class="uri">https://vowpalwabbit.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> vw
vw is /usr/local/bin/vw
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">vw</span> --help --quiet</pre>
</div>
<div id="wc" class="section level2 unnumbered">
<h2 class="unnumbered">wc</h2>
<p>Print newline, word, and byte counts for each file.
<code>wc</code>
(version 8.30)
by Paul Rubin and David MacKenzie (2019).
More information: <a href="https://www.gnu.org/software/coreutils" class="uri">https://www.gnu.org/software/coreutils</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> wc
wc is /usr/bin/wc
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> wc</pre>
</div>
<div id="which" class="section level2 unnumbered">
<h2 class="unnumbered">which</h2>
<p>Locate a command.
<code>which</code>
(version 0.1)
by Unknown (2016).
More information: .</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> which
which is a shell builtin
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> which</pre>
</div>
<div id="xml2json" class="section level2 unnumbered">
<h2 class="unnumbered">xml2json</h2>
<p>Convert an XML input to a JSON output, using xml-mapping.
<code>xml2json</code>
(version 0.0.3)
by François Parmentier (2016).
More information: <a href="https://github.com/parmentf/xml2json" class="uri">https://github.com/parmentf/xml2json</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> xml2json
xml2json is /usr/local/bin/xml2json</pre>
</div>
<div id="xmlstarlet" class="section level2 unnumbered">
<h2 class="unnumbered">xmlstarlet</h2>
<p>Command line XML/XSLT toolkit.
<code>xmlstarlet</code>
(version 1.6.1)
by Dagobert Michelsen, Noam Postavsky, and Mikhail Grushinskiy (2019).
More information: <a href="https://sourceforge.net/projects/xmlstar" class="uri">https://sourceforge.net/projects/xmlstar</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> xmlstarlet
xmlstarlet is /usr/bin/xmlstarlet
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> xmlstarlet</pre>
</div>
<div id="xsv" class="section level2 unnumbered">
<h2 class="unnumbered">xsv</h2>
<p>A fast CSV command line toolkit written in Rust.
<code>xsv</code>
(version 0.13.0)
by Andrew Gallant (2018).
More information: <a href="https://github.com/BurntSushi/xsv" class="uri">https://github.com/BurntSushi/xsv</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> xsv
xsv is /usr/bin/xsv
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">xsv</span> --help</pre>
</div>
<div id="zcat" class="section level2 unnumbered">
<h2 class="unnumbered">zcat</h2>
<p>Decompress and concatenate files to standard output.
<code>zcat</code>
(version 1.10)
by Paul Eggert (2021).
More information: <a href="https://www.nongnu.org/zutils/zutils.html" class="uri">https://www.nongnu.org/zutils/zutils.html</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> zcat
zcat is /usr/bin/zcat
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zcat</pre>
</div>
<div id="zsh" class="section level2 unnumbered">
<h2 class="unnumbered">zsh</h2>
<p>The Z shell.
<code>zsh</code>
(version 5.8)
by Paul Falstad and Peter Stephenson (2020).
More information: <a href="https://www.zsh.org" class="uri">https://www.zsh.org</a>.</p>
<pre><span style="font-weight: bold">$</span> <span style="color: #5f8700">type</span> zsh
zsh is /usr/bin/zsh
 
<span style="font-weight: bold">$</span> <span style="color: #5f8700">man</span> zsh</pre>
<!--chapter:end:tools.Rmd-->
</div>
</div>
<div class="footnotes">
<hr />
<ol>
<li id="fn1"><p>The development of the UNIX operating system started back in 1969. It featured a command line since the beginning. The important concept of pipes, which I will discuss in <a href="#essential-concepts">Section 2.3</a>, was added in 1973.<a href="#fnref1" class="footnote-back">↩︎</a></p></li>
<li id="fn2"><p><a href="#ref-Mason2010" role="doc-biblioref"><span>“A Taxonomy of Data Science,”</span> 2010, <a href="http://www.dataists.com/2010/09/a-taxonomy-of-data-science" role="doc-biblioref">http://www.dataists.com/2010/09/a-taxonomy-of-data-science</a></a>.<a href="#fnref2" class="footnote-back">↩︎</a></p></li>
<li id="fn3"><p><a href="#ref-Patil2012" role="doc-biblioref"><em>Data Jujitsu</em> (O’Reilly Media, 2012)</a>.<a href="#fnref3" class="footnote-back">↩︎</a></p></li>
<li id="fn4"><p><a href="#ref-csvstat" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">csvstat</span> – Print Descriptive Statistics for Each Column in a <span>CSV</span> File</em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref4" class="footnote-back">↩︎</a></p></li>
<li id="fn5"><p><a href="#ref-rush" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">rush</span> – <span>R</span> One-Liners from the Shell</em>, version 0.1, 2021, <a href="https://github.com/jeroenjanssens/rush" role="doc-biblioref">https://github.com/jeroenjanssens/rush</a></a>.<a href="#fnref5" class="footnote-back">↩︎</a></p></li>
<li id="fn6"><p><a href="#ref-Shron2014" role="doc-biblioref"><em>Thinking with Data</em> (O’Reilly Media, 2014)</a>.<a href="#fnref6" class="footnote-back">↩︎</a></p></li>
<li id="fn7"><p><a href="#ref-parallel" role="doc-biblioref">Ole Tange, <em><span class="nocase">parallel</span> – Build and Execute Shell Command Lines from Standard Input in Parallel</em>, version 20161222, 2016, <a href="https://www.gnu.org/software/parallel" role="doc-biblioref">https://www.gnu.org/software/parallel</a></a>.<a href="#fnref7" class="footnote-back">↩︎</a></p></li>
<li id="fn8"><p><a href="#ref-git" role="doc-biblioref">Linus Torvalds and Junio C. Hamano, <em><span class="nocase">git</span> – the Stupid Content Tracker</em>, version 2.25.1, 2021, <a href="https://git-scm.com" role="doc-biblioref">https://git-scm.com</a></a>.<a href="#fnref8" class="footnote-back">↩︎</a></p></li>
<li id="fn9"><p>See <a href="https://top500.org/statistics/details/osfam/1/">TOP500</a> which keeps track of how many super computers run Linux.<a href="#fnref9" class="footnote-back">↩︎</a></p></li>
<li id="fn10"><p><a href="#ref-ls" role="doc-biblioref">Richard M. Stallman and David MacKenzie, <em><span class="nocase">ls</span> – List Directory Contents</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref10" class="footnote-back">↩︎</a></p></li>
<li id="fn11"><p><a href="#ref-cat" role="doc-biblioref">Torbjorn Granlund and Richard M. Stallman, <em><span class="nocase">cat</span> – Concatenate Files and Print on the Standard Output</em>, version 8.30, 2018, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref11" class="footnote-back">↩︎</a></p></li>
<li id="fn12"><p><a href="#ref-jq" role="doc-biblioref">Stephen Dolan, <em><span class="nocase">jq</span> – Command-Line <span>JSON</span> Processor</em>, version 1.6, 2021, <a href="https://stedolan.github.com/jq" role="doc-biblioref">https://stedolan.github.com/jq</a></a>.<a href="#fnref12" class="footnote-back">↩︎</a></p></li>
<li id="fn13"><p><a href="#ref-seq" role="doc-biblioref">Ulrich Drepper, <em><span class="nocase">seq</span> – Print a Sequence of Numbers</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref13" class="footnote-back">↩︎</a></p></li>
<li id="fn14"><p><a href="#ref-pwd" role="doc-biblioref">Jim Meyering, <em><span class="nocase">pwd</span> – Print Name of Working Directory</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref14" class="footnote-back">↩︎</a></p></li>
<li id="fn15"><p><a href="#ref-head" role="doc-biblioref">David MacKenzie and Jim Meyering, <em><span class="nocase">head</span> – Output the First Part of Files</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref15" class="footnote-back">↩︎</a></p></li>
<li id="fn16"><p><a href="#ref-paste" role="doc-biblioref">David M. Ihnat and David MacKenzie, <em><span class="nocase">paste</span> – Merge Lines of Files</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref16" class="footnote-back">↩︎</a></p></li>
<li id="fn17"><p><a href="#ref-bc" role="doc-biblioref">Philip A. Nelson, <em><span class="nocase">bc</span> – an Arbitrary Precision Calculator Language</em>, version 1.07.1, 2017, <a href="https://www.gnu.org/software/bc" role="doc-biblioref">https://www.gnu.org/software/bc</a></a>.<a href="#fnref17" class="footnote-back">↩︎</a></p></li>
<li id="fn18"><p><a href="#ref-raymond2003art" role="doc-biblioref">Eric S Raymond, <em>The Art of Unix Programming</em> (Addison-Wesley Professional, 2003)</a>.<a href="#fnref18" class="footnote-back">↩︎</a></p></li>
<li id="fn19"><p><a href="#ref-grep" role="doc-biblioref">Jim Meyering, <em><span class="nocase">grep</span> – Print Lines That Match Patterns</em>, version 3.4, 2019, <a href="https://www.gnu.org/software/grep" role="doc-biblioref">https://www.gnu.org/software/grep</a></a>.<a href="#fnref19" class="footnote-back">↩︎</a></p></li>
<li id="fn20"><p><a href="#ref-wc" role="doc-biblioref">Paul Rubin and David MacKenzie, <em><span class="nocase">wc</span> – Print Newline, Word, and Byte Counts for Each File</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref20" class="footnote-back">↩︎</a></p></li>
<li id="fn21"><p><a href="#ref-sort" role="doc-biblioref">Mike Haertel and Paul Eggert, <em><span class="nocase">sort</span> – Sort Lines of Text Files</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref21" class="footnote-back">↩︎</a></p></li>
<li id="fn22"><p><a href="#ref-rev" role="doc-biblioref">Karel Zak, <em><span class="nocase">rev</span> – Reverse Lines Characterwise</em>, version 2.36.1, 2021, <a href="https://www.kernel.org/pub/linux/utils/util-linux" role="doc-biblioref">https://www.kernel.org/pub/linux/utils/util-linux</a></a>.<a href="#fnref22" class="footnote-back">↩︎</a></p></li>
<li id="fn23"><p><a href="http://porkmail.org/era/unix/award.html">Some</a> consider this a useless use of <code>cat</code>, arguing that the purpose of <code>cat</code> is to concatenate files and if you’re not using it for this purpose, it’s a waste of time, and costs you a process. I think this is silly. We’ve got more important things to do!<a href="#fnref23" class="footnote-back">↩︎</a></p></li>
<li id="fn24"><p><a href="#ref-sponge" role="doc-biblioref">Colin Watson and Tollef Fog Heen, <em><span class="nocase">sponge</span> – Soak up Standard Input and Write to a File</em>, version 0.65, 2021, <a href="https://joeyh.name/code/moreutils" role="doc-biblioref">https://joeyh.name/code/moreutils</a></a>.<a href="#fnref24" class="footnote-back">↩︎</a></p></li>
<li id="fn25"><p><a href="#ref-dseq" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">dseq</span> – Generate Sequence of Dates</em>, version 0.1, 2021, <a href="https://github.com/jeroenjanssens/dsutils" role="doc-biblioref">https://github.com/jeroenjanssens/dsutils</a></a>.<a href="#fnref25" class="footnote-back">↩︎</a></p></li>
<li id="fn26"><p><a href="#ref-nl" role="doc-biblioref">Scott Bartram and David MacKenzie, <em><span class="nocase">nl</span> – Number Lines of Files</em>, version 8.30, 2020, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref26" class="footnote-back">↩︎</a></p></li>
<li id="fn27"><p><a href="#ref-mv" role="doc-biblioref">Mike Parker, David MacKenzie, and Jim Meyering, <em><span class="nocase">mv</span> – Move (Rename) Files</em>, version 8.30, 2020, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref27" class="footnote-back">↩︎</a></p></li>
<li id="fn28"><p><a href="#ref-rm" role="doc-biblioref">Paul Rubin et al., <em><span class="nocase">rm</span> – Remove Files or Directories</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref28" class="footnote-back">↩︎</a></p></li>
<li id="fn29"><p><a href="#ref-cp" role="doc-biblioref">Torbjorn Granlund, David MacKenzie, and Jim Meyering, <em><span class="nocase">cp</span> – Copy Files and Directories</em>, version 8.30, 2018, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref29" class="footnote-back">↩︎</a></p></li>
<li id="fn30"><p><a href="#ref-mkdir" role="doc-biblioref">David MacKenzie, <em><span class="nocase">mkdir</span> – Make Directories</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref30" class="footnote-back">↩︎</a></p></li>
<li id="fn31"><p><a href="#ref-man" role="doc-biblioref">John W. Eaton and Colin Watson, <em><span class="nocase">man</span> – an Interface to the System Reference Manuals</em>, version 2.9.1, 2020, <a href="https://nongnu.org/man-db" role="doc-biblioref">https://nongnu.org/man-db</a></a>.<a href="#fnref31" class="footnote-back">↩︎</a></p></li>
<li id="fn32"><p><a href="#ref-tldr" role="doc-biblioref">Owen Voke, <em><span class="nocase">tldr</span> – Collaborative Cheatsheets for Console Commands</em>, version 3.3.7, 2021, <a href="https://tldr.sh" role="doc-biblioref">https://tldr.sh</a></a>.<a href="#fnref32" class="footnote-back">↩︎</a></p></li>
<li id="fn33"><p><a href="#ref-curl" role="doc-biblioref">Daniel Stenberg, <em><span class="nocase">curl</span> – Transfer a <span>URL</span></em>, version 7.68.0, 2016, <a href="https://curl.haxx.se" role="doc-biblioref">https://curl.haxx.se</a></a>.<a href="#fnref33" class="footnote-back">↩︎</a></p></li>
<li id="fn34"><p><a href="#ref-in2csv" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">in2csv</span> – Convert Common, but Less Awesome, Tabular Data Formats to <span>CSV</span></em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref34" class="footnote-back">↩︎</a></p></li>
<li id="fn35"><p><a href="#ref-sql2csv" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">sql2csv</span> – Execute an <span>SQL</span> Query on a Database and Output the Result to a <span>CSV</span> File</em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref35" class="footnote-back">↩︎</a></p></li>
<li id="fn36"><p><a href="#ref-tar" role="doc-biblioref">John Gilmore and Jay Fenlason, <em><span class="nocase">tar</span> – an Archiving Utility</em>, version 1.30, 2014, <a href="https://www.gnu.org/software/tar" role="doc-biblioref">https://www.gnu.org/software/tar</a></a>.<a href="#fnref36" class="footnote-back">↩︎</a></p></li>
<li id="fn37"><p><a href="#ref-pup" role="doc-biblioref">Eric Chiang, <em><span class="nocase">pup</span> – Parsing HTML at the Command Line</em>, version 0.4.0, 2016, <a href="https://github.com/EricChiang/pup" role="doc-biblioref">https://github.com/EricChiang/pup</a></a>.<a href="#fnref37" class="footnote-back">↩︎</a></p></li>
<li id="fn38"><p><a href="#ref-unzip" role="doc-biblioref">Samuel H. Smith et al., <em><span class="nocase">unzip</span> – List, Test and Extract Compressed Files in a <span>ZIP</span> Archive</em>, version 6.0, 2009, <a href="http://www.info-zip.org/pub/infozip" role="doc-biblioref">http://www.info-zip.org/pub/infozip</a></a>.<a href="#fnref38" class="footnote-back">↩︎</a></p></li>
<li id="fn39"><p><a href="#ref-unrar" role="doc-biblioref">Ben Asselstine, Christian Scheurer, and Johannes Winkelmann, <em><span class="nocase">unrar</span> – Extract Files from Rar Archives</em>, version 0.0.1, 2014, <a href="http://home.gna.org/unrar" role="doc-biblioref">http://home.gna.org/unrar</a></a>.<a href="#fnref39" class="footnote-back">↩︎</a></p></li>
<li id="fn40"><p><a href="#ref-unpack" role="doc-biblioref">Patrick Brisbin, <em><span class="nocase">unpack</span> – Extract Common File Formats</em>, version 0.1, 2013, <a href="https://github.com/jeroenjanssens/dsutils" role="doc-biblioref">https://github.com/jeroenjanssens/dsutils</a></a>.<a href="#fnref40" class="footnote-back">↩︎</a></p></li>
<li id="fn41"><p><a href="#ref-shafranovich2005common" role="doc-biblioref">Yakov Shafranovich, <span>“Common Format and <span>MIME</span> Type for Comma-Separated Values <span>(CSV)</span> Files,”</span> 2005</a>.<a href="#fnref41" class="footnote-back">↩︎</a></p></li>
<li id="fn42"><p><a href="#ref-csvlook" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">csvlook</span> – Render a <span>CSV</span> File in the Console as a Markdown-Compatible, Fixed-Width Table</em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref42" class="footnote-back">↩︎</a></p></li>
<li id="fn43"><p>Available on <a href="https://github.com/r-dbi/RSQLite/blob/master/inst/db/datasets.sqlite">GitHub</a>.<a href="#fnref43" class="footnote-back">↩︎</a></p></li>
<li id="fn44"><p><a href="#ref-telnet" role="doc-biblioref">Mats Erik Andersson, Andreas Henriksson, and Christoph Biedl, <em><span class="nocase">telnet</span> – User Interface to the <span>TELNET</span> Protocol</em>, version 0.17, 1999, <a href="http://www.hcs.harvard.edu/~dholland/computers/netkit.html" role="doc-biblioref">http://www.hcs.harvard.edu/~dholland/computers/netkit.html</a></a>.<a href="#fnref44" class="footnote-back">↩︎</a></p></li>
<li id="fn45"><p>If you cannot connect to the server, because someone erased it from the archive memory, then you can always enjoy a <a href="https://www.youtube.com/results?search_query=towel.blinkenlights.nl">recording of the <code>telnet</code> session on YouTube</a>.<a href="#fnref45" class="footnote-back">↩︎</a></p></li>
<li id="fn46"><p><a href="#ref-Robbins2005" role="doc-biblioref"><em>Classic Shell Scripting</em> (O’Reilly Media, 2005)</a>.<a href="#fnref46" class="footnote-back">↩︎</a></p></li>
<li id="fn47"><p><a href="#ref-tr" role="doc-biblioref">Jim Meyering, <em><span class="nocase">tr</span> – Translate or Delete Characters</em>, version 8.30, 2018, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref47" class="footnote-back">↩︎</a></p></li>
<li id="fn48"><p><a href="#ref-grep" role="doc-biblioref">Meyering, <em><span class="nocase">grep</span> – Print Lines That Match Patterns</em></a>.<a href="#fnref48" class="footnote-back">↩︎</a></p></li>
<li id="fn49"><p><a href="#ref-sort" role="doc-biblioref">Haertel and Eggert, <em><span class="nocase">sort</span> – Sort Lines of Text Files</em></a>.<a href="#fnref49" class="footnote-back">↩︎</a></p></li>
<li id="fn50"><p><a href="#ref-uniq" role="doc-biblioref">Richard M. Stallman and David MacKenzie, <em><span class="nocase">uniq</span> – Report or Omit Repeated Lines</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref50" class="footnote-back">↩︎</a></p></li>
<li id="fn51"><p><a href="#ref-bash" role="doc-biblioref">Brian Fox and Chet Ramey, <em><span class="nocase">bash</span> – <span>GNU Bourne-Again SHell</span></em>, version 5.0.17, 2019, <a href="https://www.gnu.org/software/bash" role="doc-biblioref">https://www.gnu.org/software/bash</a></a>.<a href="#fnref51" class="footnote-back">↩︎</a></p></li>
<li id="fn52"><p><a href="#ref-chmod" role="doc-biblioref">David MacKenzie and Jim Meyering, <em><span class="nocase">chmod</span> – Change File Mode Bits</em>, version 8.30, 2018, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref52" class="footnote-back">↩︎</a></p></li>
<li id="fn53"><p><a href="#ref-nano" role="doc-biblioref">Benno Schulenberg et al., <em><span class="nocase">nano</span> – <span class="nocase">Nano’s ANOther editor, inspired by Pico</span></em>, version 5.4, 2020, <a href="https://nano-editor.org" role="doc-biblioref">https://nano-editor.org</a></a>.<a href="#fnref53" class="footnote-back">↩︎</a></p></li>
<li id="fn54"><p><a href="#ref-python" role="doc-biblioref">The Python Software Foundation, <em><span class="nocase">python</span> – an Interpreted, Interactive, Object-Oriented Programming Language</em>, version 3.8.5, 2021, <a href="https://www.python.org" role="doc-biblioref">https://www.python.org</a></a>.<a href="#fnref54" class="footnote-back">↩︎</a></p></li>
<li id="fn55"><p><a href="#ref-env" role="doc-biblioref">Richard Mlynarik, David MacKenzie, and Assaf Gordon, <em><span class="nocase">env</span> – Run a Program in a Modified Environment</em>, version 8.32, 2020, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref55" class="footnote-back">↩︎</a></p></li>
<li id="fn56"><p><a href="#ref-R" role="doc-biblioref">The R Foundation for Statistical Computing, <em><span>R</span> – a Language and Environment for Statistical Computing</em>, version 4.0.4, 2021, <a href="https://www.r-project.org" role="doc-biblioref">https://www.r-project.org</a></a>.<a href="#fnref56" class="footnote-back">↩︎</a></p></li>
<li id="fn57"><p><a href="#ref-Perkins2010" role="doc-biblioref">Jacob Perkins, <em>Python Text Processing with NLTK 2.0 Cookbook</em> (Packt Publishing, 2010)</a>.<a href="#fnref57" class="footnote-back">↩︎</a></p></li>
<li id="fn58"><p><a href="#ref-McKinney2012" role="doc-biblioref">Wes McKinney, <em>Python for Data Analysis</em> (O’Reilly Media, 2012)</a>.<a href="#fnref58" class="footnote-back">↩︎</a></p></li>
<li id="fn59"><p>This code is adapted from a <a href="https://github.com/joelgrus/fizzbuzz/blob/master/fizzbuzz/cycle_of_15.py">Python script by Joel Grus</a>.<a href="#fnref59" class="footnote-back">↩︎</a></p></li>
<li id="fn60"><p><a href="#ref-grep" role="doc-biblioref">Meyering, <em><span class="nocase">grep</span> – Print Lines That Match Patterns</em></a>.<a href="#fnref60" class="footnote-back">↩︎</a></p></li>
<li id="fn61"><p><a href="#ref-awk" role="doc-biblioref">Mike D. Brennan and Thomas E. Dickey, <em><span class="nocase">awk</span> – Pattern Scanning and Text Processing Language</em>, version 1.3.4, 2019, <a href="https://invisible-island.net/mawk" role="doc-biblioref">https://invisible-island.net/mawk</a></a>.<a href="#fnref61" class="footnote-back">↩︎</a></p></li>
<li id="fn62"><p><a href="#ref-jq" role="doc-biblioref">Dolan, <em><span class="nocase">jq</span> – Command-Line <span>JSON</span> Processor</em></a>.<a href="#fnref62" class="footnote-back">↩︎</a></p></li>
<li id="fn63"><p><a href="#ref-pup" role="doc-biblioref">Chiang, <em><span class="nocase">pup</span> – Parsing HTML at the Command Line</em></a>.<a href="#fnref63" class="footnote-back">↩︎</a></p></li>
<li id="fn64"><p><a href="#ref-uniq" role="doc-biblioref">Stallman and MacKenzie, <em><span class="nocase">uniq</span> – Report or Omit Repeated Lines</em></a>.<a href="#fnref64" class="footnote-back">↩︎</a></p></li>
<li id="fn65"><p><a href="#ref-rush" role="doc-biblioref">Janssens, <em><span class="nocase">rush</span> – <span>R</span> One-Liners from the Shell</em></a>.<a href="#fnref65" class="footnote-back">↩︎</a></p></li>
<li id="fn66"><p><a href="#ref-plaintext" role="doc-biblioref">The Linux Information Project, <span>“Plain Text Definition,”</span> 2007, <a href="http://www.linfo.org/plain_text.html" role="doc-biblioref">http://www.linfo.org/plain_text.html</a></a>.<a href="#fnref66" class="footnote-back">↩︎</a></p></li>
<li id="fn67"><p><a href="#ref-pragmaticprogrammer" role="doc-biblioref">Andrew Hunt and David Thomas, <em>The Pragmatic Programmer</em> (Addison-Wesley, 1999)</a>.<a href="#fnref67" class="footnote-back">↩︎</a></p></li>
<li id="fn68"><p><a href="#ref-raymond2003art" role="doc-biblioref">Raymond, <em>The Art of Unix Programming</em></a>.<a href="#fnref68" class="footnote-back">↩︎</a></p></li>
<li id="fn69"><p><a href="#ref-head" role="doc-biblioref">MacKenzie and Meyering, <em><span class="nocase">head</span> – Output the First Part of Files</em></a>.<a href="#fnref69" class="footnote-back">↩︎</a></p></li>
<li id="fn70"><p><a href="#ref-sed" role="doc-biblioref">Jay Fenlason et al., <em><span class="nocase">sed</span> – Stream Editor for Filtering and Transforming Text</em>, version 4.7, 2018, <a href="https://www.gnu.org/software/sed" role="doc-biblioref">https://www.gnu.org/software/sed</a></a>.<a href="#fnref70" class="footnote-back">↩︎</a></p></li>
<li id="fn71"><p><a href="#ref-tail" role="doc-biblioref">Paul Rubin et al., <em><span class="nocase">tail</span> – Output the Last Part of Files</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref71" class="footnote-back">↩︎</a></p></li>
<li id="fn72"><p><a href="#ref-sample" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">sample</span> – Filter Lines from Standard Input According to Some Probability, with a Given Delay, and for a Certain Duration</em>, version 0.2.4, 2021, <a href="https://github.com/jeroenjanssens/sample" role="doc-biblioref">https://github.com/jeroenjanssens/sample</a></a>.<a href="#fnref72" class="footnote-back">↩︎</a></p></li>
<li id="fn73"><p><a href="#ref-ts" role="doc-biblioref">Joey Hess, <em><span class="nocase">ts</span> – Timestamp Input</em>, version 0.65, 2021, <a href="https://joeyh.name/code/moreutils" role="doc-biblioref">https://joeyh.name/code/moreutils</a></a>.<a href="#fnref73" class="footnote-back">↩︎</a></p></li>
<li id="fn74"><p><a href="#ref-tr" role="doc-biblioref">Meyering, <em><span class="nocase">tr</span> – Translate or Delete Characters</em></a>.<a href="#fnref74" class="footnote-back">↩︎</a></p></li>
<li id="fn75"><p><a href="#ref-body" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">body</span> – Apply Command to All but the First Line</em>, version 0.1, 2021, <a href="https://github.com/jeroenjanssens/dsutils" role="doc-biblioref">https://github.com/jeroenjanssens/dsutils</a></a>.<a href="#fnref75" class="footnote-back">↩︎</a></p></li>
<li id="fn76"><p><a href="#ref-header" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">header</span> – Add, Replace, and Delete Header Lines</em>, version 0.1, 2021, <a href="https://github.com/jeroenjanssens/dsutils" role="doc-biblioref">https://github.com/jeroenjanssens/dsutils</a></a>.<a href="#fnref76" class="footnote-back">↩︎</a></p></li>
<li id="fn77"><p><a href="#ref-cols" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">cols</span> – Apply Command to Subset of Columns</em>, version 0.1, 2021, <a href="https://github.com/jeroenjanssens/dsutils" role="doc-biblioref">https://github.com/jeroenjanssens/dsutils</a></a>.<a href="#fnref77" class="footnote-back">↩︎</a></p></li>
<li id="fn78"><p><a href="#ref-csvsql" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">csvsql</span> – Execute <span>SQL</span> Statements on <span>CSV</span> Files</em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref78" class="footnote-back">↩︎</a></p></li>
<li id="fn79"><p><a href="#ref-csvcut" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">csvcut</span> – Filter and Truncate <span>CSV</span> Files</em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref79" class="footnote-back">↩︎</a></p></li>
<li id="fn80"><p><a href="#ref-csvgrep" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">csvgrep</span> – Search <span>CSV</span> Files</em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref80" class="footnote-back">↩︎</a></p></li>
<li id="fn81"><p><a href="#ref-tee" role="doc-biblioref">Mike Parker, Richard M. Stallman, and David MacKenzie, <em><span class="nocase">tee</span> – Read from Standard Input and Write to Standard Output and Files</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref81" class="footnote-back">↩︎</a></p></li>
<li id="fn82"><p><a href="#ref-paste" role="doc-biblioref">Ihnat and MacKenzie, <em><span class="nocase">paste</span> – Merge Lines of Files</em></a>.<a href="#fnref82" class="footnote-back">↩︎</a></p></li>
<li id="fn83"><p><a href="#ref-csvjoin" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">csvjoin</span> – Execute a <span>SQL</span>-Like Join to Merge <span>CSV</span> Files on a Specified Column or Columns</em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref83" class="footnote-back">↩︎</a></p></li>
<li id="fn84"><p><a href="#ref-pup" role="doc-biblioref">Chiang, <em><span class="nocase">pup</span> – Parsing HTML at the Command Line</em></a>.<a href="#fnref84" class="footnote-back">↩︎</a></p></li>
<li id="fn85"><p><a href="#ref-xml2json" role="doc-biblioref">François Parmentier, <em><span class="nocase">xml2json</span> – Convert an <span>XML</span> Input to a <span>JSON</span> Output, Using <span class="nocase">xml-mapping</span></em>, version 0.0.3, 2016, <a href="https://github.com/parmentf/xml2json" role="doc-biblioref">https://github.com/parmentf/xml2json</a></a>.<a href="#fnref85" class="footnote-back">↩︎</a></p></li>
<li id="fn86"><p><a href="#ref-json2csv" role="doc-biblioref">Jehiah Czebotar, <em><span class="nocase">json2csv</span> – Convert <span>JSON</span> to <span>CSV</span></em>, version 1.2.1, 2019, <a href="https://github.com/jehiah/json2csv" role="doc-biblioref">https://github.com/jehiah/json2csv</a></a>.<a href="#fnref86" class="footnote-back">↩︎</a></p></li>
<li id="fn87"><p><a href="#ref-make" role="doc-biblioref">Stuart I. Feldman, <em><span class="nocase">make</span> – <span>A</span> Program for Maintaining Computer Programs</em>, version 4.3, 2020, <a href="https://www.gnu.org/software/make" role="doc-biblioref">https://www.gnu.org/software/make</a></a>.<a href="#fnref87" class="footnote-back">↩︎</a></p></li>
<li id="fn88"><p><a href="#ref-drake" role="doc-biblioref">Factual, <em><span class="nocase">drake</span> – Data Workflow Tool, Like a "Make for Data"</em>, version 1.0.3, 2016, <a href="https://github.com/Factual/drake" role="doc-biblioref">https://github.com/Factual/drake</a></a>.<a href="#fnref88" class="footnote-back">↩︎</a></p></li>
<li id="fn89"><p><a href="#ref-git" role="doc-biblioref">Torvalds and Hamano, <em><span class="nocase">git</span> – the Stupid Content Tracker</em></a>.<a href="#fnref89" class="footnote-back">↩︎</a></p></li>
<li id="fn90"><p><a href="#ref-less" role="doc-biblioref">Mark Nudelman, <em><span class="nocase">less</span> – Opposite of <span class="nocase">more</span></em>, version 551, 2019, <a href="https://www.greenwoodsoftware.com/less" role="doc-biblioref">https://www.greenwoodsoftware.com/less</a></a>.<a href="#fnref90" class="footnote-back">↩︎</a></p></li>
<li id="fn91"><p><a href="#ref-csvstat" role="doc-biblioref">Groskopf, <em><span class="nocase">csvstat</span> – Print Descriptive Statistics for Each Column in a <span>CSV</span> File</em></a>.<a href="#fnref91" class="footnote-back">↩︎</a></p></li>
<li id="fn92"><p><a href="#ref-rush" role="doc-biblioref">Janssens, <em><span class="nocase">rush</span> – <span>R</span> One-Liners from the Shell</em></a>.<a href="#fnref92" class="footnote-back">↩︎</a></p></li>
<li id="fn93"><p><a href="#ref-R" role="doc-biblioref">The R Foundation for Statistical Computing, <em><span>R</span> – a Language and Environment for Statistical Computing</em></a>.<a href="#fnref93" class="footnote-back">↩︎</a></p></li>
<li id="fn94"><p><a href="#ref-servewd" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">servewd</span> – Serve the Current Working Directory Using a Simple <span>HTTP</span> Server</em>, version 0.1, 2021, <a href="https://github.com/jeroenjanssens/dsutils" role="doc-biblioref">https://github.com/jeroenjanssens/dsutils</a></a>.<a href="#fnref94" class="footnote-back">↩︎</a></p></li>
<li id="fn95"><p><a href="#ref-parallel" role="doc-biblioref">Tange, <em><span class="nocase">parallel</span> – Build and Execute Shell Command Lines from Standard Input in Parallel</em></a>.<a href="#fnref95" class="footnote-back">↩︎</a></p></li>
<li id="fn96"><p><a href="#ref-bc" role="doc-biblioref">Nelson, <em><span class="nocase">bc</span> – an Arbitrary Precision Calculator Language</em></a>.<a href="#fnref96" class="footnote-back">↩︎</a></p></li>
<li id="fn97"><p><a href="#ref-ls" role="doc-biblioref">Stallman and MacKenzie, <em><span class="nocase">ls</span> – List Directory Contents</em></a>.<a href="#fnref97" class="footnote-back">↩︎</a></p></li>
<li id="fn98"><p><a href="#ref-find" role="doc-biblioref">Eric B. Decker, James Youngman, and Kevin Dalley, <em><span class="nocase">find</span> – Search for Files in a Directory Hierarchy</em>, version 4.7.0, 2019, <a href="https://www.gnu.org/software/findutils" role="doc-biblioref">https://www.gnu.org/software/findutils</a></a>.<a href="#fnref98" class="footnote-back">↩︎</a></p></li>
<li id="fn99"><p><a href="#ref-ts" role="doc-biblioref">Hess, <em><span class="nocase">ts</span> – Timestamp Input</em></a>.<a href="#fnref99" class="footnote-back">↩︎</a></p></li>
<li id="fn100"><p><a href="#ref-pbc" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">pbc</span> – Parallel <span class="nocase">bc</span></em>, version 0.1, 2021, <a href="https://github.com/jeroenjanssens/dsutils" role="doc-biblioref">https://github.com/jeroenjanssens/dsutils</a></a>.<a href="#fnref100" class="footnote-back">↩︎</a></p></li>
<li id="fn101"><p><a href="#ref-aws" role="doc-biblioref">Amazon Web Services, <em><span class="nocase">aws</span> – Unified Tool to Manage <span>AWS</span> Services</em>, version 2.1.32, 2021, <a href="https://aws.amazon.com/cli" role="doc-biblioref">https://aws.amazon.com/cli</a></a>.<a href="#fnref101" class="footnote-back">↩︎</a></p></li>
<li id="fn102"><p><a href="#ref-ssh" role="doc-biblioref">Tatu Ylonen et al., <em><span class="nocase">ssh</span> – <span>OpenSSH</span> Remote Login Client</em>, version 1:8.2p1-4ubuntu0.2, 2020, <a href="https://www.openssh.com" role="doc-biblioref">https://www.openssh.com</a></a>.<a href="#fnref102" class="footnote-back">↩︎</a></p></li>
<li id="fn103"><p><a href="#ref-hostname" role="doc-biblioref">Peter Tobias, Bernd Eckenfels, and Michael Meskes, <em><span class="nocase">hostname</span> – Show or Set the System’s Host Name</em>, version 3.23, 2021, <a href="https://sourceforge.net/projects/net-tools/" role="doc-biblioref">https://sourceforge.net/projects/net-tools/</a></a>.<a href="#fnref103" class="footnote-back">↩︎</a></p></li>
<li id="fn104"><p><a href="#ref-zcat" role="doc-biblioref">Paul Eggert, <em><span class="nocase">zcat</span> – Decompress and Concatenate Files to Standard Output</em>, version 1.10, 2021, <a href="https://www.nongnu.org/zutils/zutils.html" role="doc-biblioref">https://www.nongnu.org/zutils/zutils.html</a></a>.<a href="#fnref104" class="footnote-back">↩︎</a></p></li>
<li id="fn105"><p><a href="#ref-rush" role="doc-biblioref">Janssens, <em><span class="nocase">rush</span> – <span>R</span> One-Liners from the Shell</em></a>.<a href="#fnref105" class="footnote-back">↩︎</a></p></li>
<li id="fn106"><p><a href="#ref-tapkee" role="doc-biblioref">Sergey Lisitsyn, Christian Widmer, and Fernando J. Iglesias Garcia, <em><span class="nocase">tapkee</span> – an Efficient Dimension Reduction Library</em>, version 1.2, 2013, <a href="http://tapkee.lisitsyn.me" role="doc-biblioref">http://tapkee.lisitsyn.me</a></a>.<a href="#fnref106" class="footnote-back">↩︎</a></p></li>
<li id="fn107"><p><a href="#ref-vw" role="doc-biblioref">John Langford, <em><span class="nocase">vw</span> – Fast Machine Learning Library for Online Learning</em>, version 8.10.1, 2021, <a href="https://vowpalwabbit.org" role="doc-biblioref">https://vowpalwabbit.org</a></a>.<a href="#fnref107" class="footnote-back">↩︎</a></p></li>
<li id="fn108"><p><a href="#ref-skll" role="doc-biblioref">Educational Testing Service, <em><span class="nocase">skll</span> – <span>SciKit-Learn</span> Laboratory</em>, version 2.5.0, 2021, <a href="https://skll.readthedocs.org" role="doc-biblioref">https://skll.readthedocs.org</a></a>.<a href="#fnref108" class="footnote-back">↩︎</a></p></li>
<li id="fn109"><p><a href="#ref-csvstack" role="doc-biblioref">Christopher Groskopf, <em><span class="nocase">csvstack</span> – Stack up the Rows from Multiple <span>CSV</span> Files</em>, version 1.0.5, 2020, <a href="https://csvkit.rtfd.org" role="doc-biblioref">https://csvkit.rtfd.org</a></a>.<a href="#fnref109" class="footnote-back">↩︎</a></p></li>
<li id="fn110"><p><a href="#ref-Pearson1901" role="doc-biblioref">K. Pearson, <span>“On Lines and Planes of Closest Fit to Systems of Points in Space,”</span> <em>Philosophical Magazine</em> 2, no. 11 (1901): 559–72</a>.<a href="#fnref110" class="footnote-back">↩︎</a></p></li>
<li id="fn111"><p><a href="#ref-van2008visualizing" role="doc-biblioref">Laurens van der Maaten and Geoffrey Everest Hinton, <span>“Visualizing Data Using t-<span>SNE</span>,”</span> <em>Journal of Machine Learning Research</em> 9 (2008): 2579–2605</a>.<a href="#fnref111" class="footnote-back">↩︎</a></p></li>
<li id="fn112"><p><a href="#ref-Lisitsyn2013" role="doc-biblioref">Sergey Lisitsyn, Christian Widmer, and Fernando J. Iglesias Garcia, <span>“Tapkee: An Efficient Dimension Reduction Library,”</span> <em>Journal of Machine Learning Research</em> 14 (2013): 2355–59</a>.<a href="#fnref112" class="footnote-back">↩︎</a></p></li>
<li id="fn113"><p><a href="#ref-csv2vw" role="doc-biblioref">Jeroen Janssens, <em><span class="nocase">csv2vw</span> – Convert <span>CSV</span> to Vowpal Wabbit Format</em>, version 0.1, 2021, <a href="https://github.com/jeroenjanssens/dsutils" role="doc-biblioref">https://github.com/jeroenjanssens/dsutils</a></a>.<a href="#fnref113" class="footnote-back">↩︎</a></p></li>
<li id="fn114"><p><a href="#ref-split" role="doc-biblioref">Torbjorn Granlund and Richard M. Stallman, <em><span class="nocase">split</span> – Split a File into Pieces</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref114" class="footnote-back">↩︎</a></p></li>
<li id="fn115"><p><a href="#ref-shuf" role="doc-biblioref">Paul Eggert, <em><span class="nocase">shuf</span> – Generate Random Permutations</em>, version 8.30, 2019, <a href="https://www.gnu.org/software/coreutils" role="doc-biblioref">https://www.gnu.org/software/coreutils</a></a>.<a href="#fnref115" class="footnote-back">↩︎</a></p></li>
</ol>
</div>
<!--bookdown:body:end-->
  </main>

  <div class="col-md-3 col-lg-3 d-none d-md-block sidebar sidebar-chapter">
    <nav id="toc" data-toggle="toc" aria-label="On this page">
      <h2>On this page</h2>
      <div id="book-on-this-page"></div>

      <div class="book-extra">
        <ul class="list-unstyled">
          <li><a id="book-source" href="#">View source <i class="fab fa-github"></i></a></li>
          <li><a id="book-edit" href="#">Edit this page <i class="fab fa-github"></i></a></li>
        </ul>
      </div>
    </nav>
  </div>

</div>
</div> <!-- .container -->

<footer class="bg-primary text-light mt-5">
  <div class="container-fluid">
    <div class="row">
      <div class="d-none d-lg-block col-lg-2 sidebar"></div>
      <div class="col-sm-12 col-md-9 col-lg-7 mt-3" style="max-width: 45rem;">
        <p><strong>Data Science at the Command Line, 2e</strong> by <a href="https://twitter.com/jeroenhjanssens" class="text-light">Jeroen Janssens</a>. Updated on December 14, 2021. This book was built by the <a class="text-light" href="https://bookdown.org">bookdown</a> R package.</p>
      </div>
      <div class="col-md-3 col-lg-3 d-none d-md-block sidebar"></div>
    </div>
  </div>
</footer>


</body>

</html>
